diff --git a/3_AZALEE/insert_ps.py b/3_AZALEE/insert_ps.py index 76c419b..f338edd 100644 --- a/3_AZALEE/insert_ps.py +++ b/3_AZALEE/insert_ps.py @@ -8,9 +8,9 @@ # import re from contextlib import AsyncExitStack -from os import register_at_fork +from os import register_at_fork,path import types -from numpy.lib.shape_base import expand_dims +# from numpy.lib.shape_base import expand_dims import pandas as pd import geopandas as gpd from pandas.io.pytables import incompatibility_doc @@ -340,6 +340,7 @@ def normalize_colname(df): 'remarque' :'remarques', 'remarq_' :'remarques', 'rmq_intere' :'remarques', + # 'rmq_site' :'remarques', 'legendes' :'legende', 'legend' :'legende', 'sources' :'source', @@ -398,6 +399,7 @@ def existing_ps(df): return df + def define_siteOrg(df): gdf = pycen.get_districtNat()[['abrev','geom']] tmp = df[['geom']].copy().set_geometry('geom') @@ -415,6 +417,7 @@ def define_siteOrg(df): return df # tmp[tmp.id_site==334] + # 'touches', None, 'contains', 'overlaps', 'contains_properly', 'within', 'intersects', 'crosses', 'covers', 'covered_by' def ident_newsite(df,rcvmt=10): ''' @@ -467,7 +470,7 @@ def ident_newsite(df,rcvmt=10): tmpp[['site_code','id_origine']], on=['id_origine']) maj2['id_site'] = maj2['site_code'] del maj2['site_code'] - if maj : df_maj = pd.concat([maj,maj2]) + if not maj.empty : df_maj = pd.concat([maj,maj2]) else : df_maj = maj2 # Isolement des nouveaux sites orig_maj_all = df_maj.id_origine @@ -483,7 +486,6 @@ def ident_newsite(df,rcvmt=10): return df_new, df_cor, df_maj - def define_siteName(df,rcvmt=10): ''' Return df_new, df_replace, df_maj @@ -536,7 +538,6 @@ def define_siteName(df,rcvmt=10): return df_new, df_replace, df_maj - def normalize_formatIdsite(id_site): import re df = pd.DataFrame({'id_site':id_site}) @@ -753,12 +754,12 @@ def insert_site(df): df['type_site'] = 'N.D.' if 'type_milieu' not in df.columns: df['type_milieu'] = 'Pelouses sèches' - # if 'remarques' not in df.columns: - # df['remarques'] = None + if 'remarques' not in df.columns: + df['remarques'] = None df = df[['id_site','nom','date','auteur','type_site','type_milieu', 'structure' - # 'remarques' + ,'remarques' ]] drop_col = ['auteur','structure'] typs = pycen.sites._get_typ_site() @@ -836,7 +837,7 @@ def insert_geom(df): colindict = [ *df.columns[df.columns.isin(dic.keys())] ] if colindict: df.rename( - columns={'remarques':'rmq_interet_patri','legende':'rmq_usage_process'}, + columns=dic, inplace=True, ) if 'table_org' in df.columns: @@ -1166,15 +1167,20 @@ def filter_saisierror(df): continue print('\t'+cod + ' : '+ perc.astype(str)) lst_error1 = [*lst_error1,*tmp[(tmp.hab_ == hab)&(tmp.code_hab==cod)].sort_values('id_site').id_site.unique()] - df[df.id_site.isin(lst_error1)].to_postgis( - name='incoherence_habVScodehab', - con=con, - schema=schema, - if_exists='replace', - index=False, - # geom_col='geom', - index_label='id_site', - ) + try: + df[df.id_site.isin(lst_error1)].to_postgis( + name='incoherence_habVScodehab', + con=con, + schema=schema, + if_exists='replace', + index=False, + # geom_col='geom', + index_label='id_site', + ) + except Exception as e: + print(e) + df[df.id_site.isin(lst_error1)].to_file(path.join(path0,'incoherence_habVScodehab.geojson'),driver='GeoJSON') + if lst_error1: print('%i'%len(lst_error1)) @@ -1196,42 +1202,51 @@ def filter_saisierror(df): if lst_error2: print('ERROR code_hab : %i'%len(lst_error2)) - df[df.id_site.isin(lst_error2)].to_postgis( - name='PB_codehab_nonPresent_dans_corineBiotope', - con=con, - schema=schema, - if_exists='append', - index=False, - # geom_col='geom', - index_label='id_site', - ) + try: + df[df.id_site.isin(lst_error2)].to_postgis( + name='PB_codehab_nonPresent_dans_corineBiotope', + con=con, + schema=schema, + if_exists='append', + index=False, + # geom_col='geom', + index_label='id_site', + ) + except Exception as e: + print(e) + df[df.id_site.isin(lst_error2)].to_file(path.join(path0,'PB_codehab_nonPresent_dans_corineBiotope.geojson'),driver='GeoJSON') lst_iderror = [*lst_error1,*lst_error2] lst_iderrorindex = df[df.id_site.isin(lst_iderror)].index df.drop(lst_iderrorindex,inplace=True) return df.sort_values('id_site') -def _insert_site_(df,ps_statut='actif'): + +def _insert_geomsite(df): df_ins, ins = insert_geom(df) if ins: insert_authordata(df_ins,id_data='id_site',tab_data='r_sites_geom',colname_rsite='id_geom_site', tab_auteur='r_geomsites_auteur') - - df_ins, ins = insert_attrsPS(df,ps_statut) - if ins: - insert_authordata(df_ins,id_data='id_geom_site',tab_data='r_site_param',colname_rsite='id_siteparam', tab_auteur='r_siteparam_auteur') - df_ins, ins = insert_habPS(df,ps_statut) - if ins: - if df_ins.shape[0] > 4000: - df_ins1 = df_ins.iloc[:round(df_ins.shape[0]/2)].copy() - df_ins2 = df_ins.iloc[round(df_ins.shape[0]/2):].copy() - insert_cbPS(df_ins1) - insert_cbPS(df_ins2) - else: - insert_cbPS(df_ins) - insert_authordata( - df_ins[df_ins.columns.drop('code_hab')],id_data='id_geom_site',tab_data='r_site_habitat', - colname_rsite='id_sitehab', tab_auteur='r_sitehab_auteur') + +def _insert_site_(df,ps_statut='actif'): + _insert_geomsite(df) + + df_ins, ins = insert_attrsPS(df,ps_statut) + if ins: + insert_authordata(df_ins,id_data='id_geom_site',tab_data='r_site_param',colname_rsite='id_siteparam', tab_auteur='r_siteparam_auteur') + + df_ins, ins = insert_habPS(df,ps_statut) + if ins: + if df_ins.shape[0] > 4000: + df_ins1 = df_ins.iloc[:round(df_ins.shape[0]/2)].copy() + df_ins2 = df_ins.iloc[round(df_ins.shape[0]/2):].copy() + insert_cbPS(df_ins1) + insert_cbPS(df_ins2) + else: + insert_cbPS(df_ins) + insert_authordata( + df_ins[df_ins.columns.drop('code_hab')],id_data='id_geom_site',tab_data='r_site_habitat', + colname_rsite='id_sitehab', tab_auteur='r_sitehab_auteur') def maj_site(df): @@ -1305,7 +1320,7 @@ if __name__ == "__main__": from_table = None from_file = 'PS prioritaire.geojson' from_tab = 'Tableau_saisie_PS_2026.xlsx' - path0 = '/home/colas/Documents/9_PROJETS/2_PS/TO IMPORT/2026/' + path0 = '/home/cgeier/Documents/9_PROJETS/2_PS/TO IMPORT/2026' # org = from_file.split('/')[1] tutu = pd.DataFrame() @@ -1354,11 +1369,13 @@ if __name__ == "__main__": if from_file is not None: table = from_file.split('/')[-1]#.split('.')[0] - df = gpd.read_file(path0+from_file) + df = gpd.read_file(path.join(path0,from_file)) + df.dropna(axis=1, how='all', inplace=True) df = normalize_colname(df) df = format_date(df) df['table_org'] = table df['structure'] = 'CEN Isère' + df['structure'] = df.organisme.copy() # df['type_pat'].replace(['Indéterminé','/'],None,inplace=True) # df.loc[df.type_pat.notna(),'pratiques'] = \ # df[df.type_pat.notna()].pratiques + ' ' + df[df.type_pat.notna()].type_pat.str.lower() @@ -1384,14 +1401,14 @@ if __name__ == "__main__": tutu.rename(columns={'ident':'id_origine'}, inplace=True) cols = tutu.columns[tutu.columns.isin(c)] df = tutu[cols].copy() - df = df.set_geometry('geom',crs=2154) + df = df.set_geometry('geom', crs=2154) # df.code_hab3.replace({ # '43.171':'41.171', # '31.4B':'31.4' # }, inplace=True) df = pycen.tools.Polygons_to_MultiPolygon(df) - df_new, df_replace, df_maj = define_siteName(df,rcvmt=10) + df_new, df_replace, df_maj = define_siteName(df, rcvmt=5) df_new_tmp = df_new.copy() df_new = df_new_tmp[df_new_tmp.id_site.notna()].copy() diff --git a/3_AZALEE/update_psInf1200.py b/3_AZALEE/update_psInf1200.py index 6fbca03..bde4fe8 100644 --- a/3_AZALEE/update_psInf1200.py +++ b/3_AZALEE/update_psInf1200.py @@ -7,34 +7,35 @@ import geopandas as gpd def update_psInf1200(df): - from rasterstats import zonal_stats - path0 = '/home/colas/Documents/9_PROJETS/2_PS/' + path0 = '/home/cgeier/Documents/9_PROJETS/2_PS/' path2_mnt = path0+'MNT/' - file_mnt5 = 'MNT_5m.tif' + file_mnt5 = 'MNT_5m.tif' zonal_tmp = zonal_stats(df,path2_mnt+file_mnt5,stats="max",nodata=0,all_touched=True) zonal = gpd.pd.DataFrame(zonal_tmp) zonal.columns = ['max_alti'] tmp = gpd.pd.concat([df.reset_index(drop=True),zonal], axis=1) tmp = tmp.set_geometry('geom', crs=2154) - tmp.infeq_1200 = tmp.max_alti <= 1200 + tmp['infeq_1200'] = tmp.max_alti <= 1200 - PS = tmp[['site_code','infeq_1200']]\ - .rename(columns={'site_code':'id_site'}) + PS = (tmp[['site_code','infeq_1200']] + .rename(columns={'site_code':'id_site'},errors='ignore') + ) PS.to_sql('r_infeq_1200m',con,'ps',if_exists='append',index=False) # update_to_sql(PS,con,'r_infeq_1200m','ps','id_site') - ps_all = gpd.read_postgis(''' SELECT vp.*,s7.infeq_1200 FROM ps."v_pelouseseches_all" vp LEFT JOIN ps.r_infeq_1200m s7 ON vp.site_code::text = s7.id_site::text ORDER BY vp.site_code DESC NULLS LAST; -''',con) +''', con) isnot_1200 = ps_all.infeq_1200.isna() +ps_all[~ps_all.infeq_1200] + update_psInf1200(ps_all[isnot_1200])