CEN38/Python_scripts
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update code

Browse Source
This commit is contained in:
Colas Geier 2026-07-21 11:36:17 +02:00
parent 255202d195
commit ca84adc12e
2 changed files with 74 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
3_AZALEE/insert_ps.py
View File

@ -8,9 +8,9 @@
# import re # import re
from contextlib import AsyncExitStack from contextlib import AsyncExitStack
from os import register_at_fork from os import register_at_fork,path
import types import types
from numpy.lib.shape_base import expand_dims # from numpy.lib.shape_base import expand_dims
import pandas as pd import pandas as pd
import geopandas as gpd import geopandas as gpd
from pandas.io.pytables import incompatibility_doc from pandas.io.pytables import incompatibility_doc
@ -340,6 +340,7 @@ def normalize_colname(df):
'remarque' :'remarques', 'remarque' :'remarques',
'remarq_' :'remarques', 'remarq_' :'remarques',
'rmq_intere' :'remarques', 'rmq_intere' :'remarques',
# 'rmq_site' :'remarques',
'legendes' :'legende', 'legendes' :'legende',
'legend' :'legende', 'legend' :'legende',
'sources' :'source', 'sources' :'source',
@ -398,6 +399,7 @@ def existing_ps(df):
return df return df
def define_siteOrg(df): def define_siteOrg(df):
gdf = pycen.get_districtNat()[['abrev','geom']] gdf = pycen.get_districtNat()[['abrev','geom']]
tmp = df[['geom']].copy().set_geometry('geom') tmp = df[['geom']].copy().set_geometry('geom')
@ -415,6 +417,7 @@ def define_siteOrg(df):
return df return df
# tmp[tmp.id_site==334] # tmp[tmp.id_site==334]
# 'touches', None, 'contains', 'overlaps', 'contains_properly', 'within', 'intersects', 'crosses', 'covers', 'covered_by' # 'touches', None, 'contains', 'overlaps', 'contains_properly', 'within', 'intersects', 'crosses', 'covers', 'covered_by'
def ident_newsite(df,rcvmt=10): def ident_newsite(df,rcvmt=10):
''' '''
@ -467,7 +470,7 @@ def ident_newsite(df,rcvmt=10):
tmpp[['site_code','id_origine']], on=['id_origine']) tmpp[['site_code','id_origine']], on=['id_origine'])
maj2['id_site'] = maj2['site_code'] maj2['id_site'] = maj2['site_code']
del maj2['site_code'] del maj2['site_code']
if maj : df_maj = pd.concat([maj,maj2]) if not maj.empty : df_maj = pd.concat([maj,maj2])
else : df_maj = maj2 else : df_maj = maj2
# Isolement des nouveaux sites # Isolement des nouveaux sites
orig_maj_all = df_maj.id_origine orig_maj_all = df_maj.id_origine
@ -483,7 +486,6 @@ def ident_newsite(df,rcvmt=10):
return df_new, df_cor, df_maj return df_new, df_cor, df_maj
def define_siteName(df,rcvmt=10): def define_siteName(df,rcvmt=10):
''' '''
Return df_new, df_replace, df_maj Return df_new, df_replace, df_maj
@ -536,7 +538,6 @@ def define_siteName(df,rcvmt=10):
return df_new, df_replace, df_maj return df_new, df_replace, df_maj
def normalize_formatIdsite(id_site): def normalize_formatIdsite(id_site):
import re import re
df = pd.DataFrame({'id_site':id_site}) df = pd.DataFrame({'id_site':id_site})
@ -753,12 +754,12 @@ def insert_site(df):
df['type_site'] = 'N.D.' df['type_site'] = 'N.D.'
if 'type_milieu' not in df.columns: if 'type_milieu' not in df.columns:
df['type_milieu'] = 'Pelouses sèches' df['type_milieu'] = 'Pelouses sèches'
# if 'remarques' not in df.columns: if 'remarques' not in df.columns:
# df['remarques'] = None df['remarques'] = None
df = df[['id_site','nom','date','auteur','type_site','type_milieu', df = df[['id_site','nom','date','auteur','type_site','type_milieu',
'structure' 'structure'
# 'remarques' ,'remarques'
]] ]]
drop_col = ['auteur','structure'] drop_col = ['auteur','structure']
typs = pycen.sites._get_typ_site() typs = pycen.sites._get_typ_site()
@ -836,7 +837,7 @@ def insert_geom(df):
colindict = [ *df.columns[df.columns.isin(dic.keys())] ] colindict = [ *df.columns[df.columns.isin(dic.keys())] ]
if colindict: if colindict:
df.rename( df.rename(
columns={'remarques':'rmq_interet_patri','legende':'rmq_usage_process'}, columns=dic,
inplace=True, inplace=True,
) )
if 'table_org' in df.columns: if 'table_org' in df.columns:
@ -1166,15 +1167,20 @@ def filter_saisierror(df):
continue continue
print('\t'+cod + ' : '+ perc.astype(str)) print('\t'+cod + ' : '+ perc.astype(str))
lst_error1 = [*lst_error1,*tmp[(tmp.hab_ == hab)&(tmp.code_hab==cod)].sort_values('id_site').id_site.unique()] lst_error1 = [*lst_error1,*tmp[(tmp.hab_ == hab)&(tmp.code_hab==cod)].sort_values('id_site').id_site.unique()]
df[df.id_site.isin(lst_error1)].to_postgis( try:
name='incoherence_habVScodehab', df[df.id_site.isin(lst_error1)].to_postgis(
con=con, name='incoherence_habVScodehab',
schema=schema, con=con,
if_exists='replace', schema=schema,
index=False, if_exists='replace',
# geom_col='geom', index=False,
index_label='id_site', # geom_col='geom',
) index_label='id_site',
)
except Exception as e:
print(e)
df[df.id_site.isin(lst_error1)].to_file(path.join(path0,'incoherence_habVScodehab.geojson'),driver='GeoJSON')
if lst_error1: if lst_error1:
print('%i'%len(lst_error1)) print('%i'%len(lst_error1))
@ -1196,42 +1202,51 @@ def filter_saisierror(df):
if lst_error2: if lst_error2:
print('ERROR code_hab : %i'%len(lst_error2)) print('ERROR code_hab : %i'%len(lst_error2))
df[df.id_site.isin(lst_error2)].to_postgis( try:
name='PB_codehab_nonPresent_dans_corineBiotope', df[df.id_site.isin(lst_error2)].to_postgis(
con=con, name='PB_codehab_nonPresent_dans_corineBiotope',
schema=schema, con=con,
if_exists='append', schema=schema,
index=False, if_exists='append',
# geom_col='geom', index=False,
index_label='id_site', # geom_col='geom',
) index_label='id_site',
)
except Exception as e:
print(e)
df[df.id_site.isin(lst_error2)].to_file(path.join(path0,'PB_codehab_nonPresent_dans_corineBiotope.geojson'),driver='GeoJSON')
lst_iderror = [*lst_error1,*lst_error2] lst_iderror = [*lst_error1,*lst_error2]
lst_iderrorindex = df[df.id_site.isin(lst_iderror)].index lst_iderrorindex = df[df.id_site.isin(lst_iderror)].index
df.drop(lst_iderrorindex,inplace=True) df.drop(lst_iderrorindex,inplace=True)
return df.sort_values('id_site') return df.sort_values('id_site')
def _insert_site_(df,ps_statut='actif'):
def _insert_geomsite(df):
df_ins, ins = insert_geom(df) df_ins, ins = insert_geom(df)
if ins: if ins:
insert_authordata(df_ins,id_data='id_site',tab_data='r_sites_geom',colname_rsite='id_geom_site', tab_auteur='r_geomsites_auteur') insert_authordata(df_ins,id_data='id_site',tab_data='r_sites_geom',colname_rsite='id_geom_site', tab_auteur='r_geomsites_auteur')
df_ins, ins = insert_attrsPS(df,ps_statut)
if ins:
insert_authordata(df_ins,id_data='id_geom_site',tab_data='r_site_param',colname_rsite='id_siteparam', tab_auteur='r_siteparam_auteur')
df_ins, ins = insert_habPS(df,ps_statut) def _insert_site_(df,ps_statut='actif'):
if ins: _insert_geomsite(df)
if df_ins.shape[0] > 4000:
df_ins1 = df_ins.iloc[:round(df_ins.shape[0]/2)].copy() df_ins, ins = insert_attrsPS(df,ps_statut)
df_ins2 = df_ins.iloc[round(df_ins.shape[0]/2):].copy() if ins:
insert_cbPS(df_ins1) insert_authordata(df_ins,id_data='id_geom_site',tab_data='r_site_param',colname_rsite='id_siteparam', tab_auteur='r_siteparam_auteur')
insert_cbPS(df_ins2)
else: df_ins, ins = insert_habPS(df,ps_statut)
insert_cbPS(df_ins) if ins:
insert_authordata( if df_ins.shape[0] > 4000:
df_ins[df_ins.columns.drop('code_hab')],id_data='id_geom_site',tab_data='r_site_habitat', df_ins1 = df_ins.iloc[:round(df_ins.shape[0]/2)].copy()
colname_rsite='id_sitehab', tab_auteur='r_sitehab_auteur') df_ins2 = df_ins.iloc[round(df_ins.shape[0]/2):].copy()
insert_cbPS(df_ins1)
insert_cbPS(df_ins2)
else:
insert_cbPS(df_ins)
insert_authordata(
df_ins[df_ins.columns.drop('code_hab')],id_data='id_geom_site',tab_data='r_site_habitat',
colname_rsite='id_sitehab', tab_auteur='r_sitehab_auteur')
def maj_site(df): def maj_site(df):
@ -1305,7 +1320,7 @@ if __name__ == "__main__":
from_table = None from_table = None
from_file = 'PS prioritaire.geojson' from_file = 'PS prioritaire.geojson'
from_tab = 'Tableau_saisie_PS_2026.xlsx' from_tab = 'Tableau_saisie_PS_2026.xlsx'
path0 = '/home/colas/Documents/9_PROJETS/2_PS/TO IMPORT/2026/' path0 = '/home/cgeier/Documents/9_PROJETS/2_PS/TO IMPORT/2026'
# org = from_file.split('/')[1] # org = from_file.split('/')[1]
tutu = pd.DataFrame() tutu = pd.DataFrame()
@ -1354,11 +1369,13 @@ if __name__ == "__main__":
if from_file is not None: if from_file is not None:
table = from_file.split('/')[-1]#.split('.')[0] table = from_file.split('/')[-1]#.split('.')[0]
df = gpd.read_file(path0+from_file) df = gpd.read_file(path.join(path0,from_file))
df.dropna(axis=1, how='all', inplace=True)
df = normalize_colname(df) df = normalize_colname(df)
df = format_date(df) df = format_date(df)
df['table_org'] = table df['table_org'] = table
df['structure'] = 'CEN Isère' df['structure'] = 'CEN Isère'
df['structure'] = df.organisme.copy()
# df['type_pat'].replace(['Indéterminé','/'],None,inplace=True) # df['type_pat'].replace(['Indéterminé','/'],None,inplace=True)
# df.loc[df.type_pat.notna(),'pratiques'] = \ # df.loc[df.type_pat.notna(),'pratiques'] = \
# df[df.type_pat.notna()].pratiques + ' ' + df[df.type_pat.notna()].type_pat.str.lower() # df[df.type_pat.notna()].pratiques + ' ' + df[df.type_pat.notna()].type_pat.str.lower()
@ -1384,14 +1401,14 @@ if __name__ == "__main__":
tutu.rename(columns={'ident':'id_origine'}, inplace=True) tutu.rename(columns={'ident':'id_origine'}, inplace=True)
cols = tutu.columns[tutu.columns.isin(c)] cols = tutu.columns[tutu.columns.isin(c)]
df = tutu[cols].copy() df = tutu[cols].copy()
df = df.set_geometry('geom',crs=2154) df = df.set_geometry('geom', crs=2154)
# df.code_hab3.replace({ # df.code_hab3.replace({
# '43.171':'41.171', # '43.171':'41.171',
# '31.4B':'31.4' # '31.4B':'31.4'
# }, inplace=True) # }, inplace=True)
df = pycen.tools.Polygons_to_MultiPolygon(df) df = pycen.tools.Polygons_to_MultiPolygon(df)
df_new, df_replace, df_maj = define_siteName(df,rcvmt=10) df_new, df_replace, df_maj = define_siteName(df, rcvmt=5)
df_new_tmp = df_new.copy() df_new_tmp = df_new.copy()
df_new = df_new_tmp[df_new_tmp.id_site.notna()].copy() df_new = df_new_tmp[df_new_tmp.id_site.notna()].copy()

17
3_AZALEE/update_psInf1200.py
View File

@ -7,34 +7,35 @@ import geopandas as gpd
def update_psInf1200(df): def update_psInf1200(df):
from rasterstats import zonal_stats from rasterstats import zonal_stats
path0 = '/home/colas/Documents/9_PROJETS/2_PS/' path0 = '/home/cgeier/Documents/9_PROJETS/2_PS/'
path2_mnt = path0+'MNT/' path2_mnt = path0+'MNT/'
file_mnt5 = 'MNT_5m.tif' file_mnt5 = 'MNT_5m.tif'
zonal_tmp = zonal_stats(df,path2_mnt+file_mnt5,stats="max",nodata=0,all_touched=True) zonal_tmp = zonal_stats(df,path2_mnt+file_mnt5,stats="max",nodata=0,all_touched=True)
zonal = gpd.pd.DataFrame(zonal_tmp) zonal = gpd.pd.DataFrame(zonal_tmp)
zonal.columns = ['max_alti'] zonal.columns = ['max_alti']
tmp = gpd.pd.concat([df.reset_index(drop=True),zonal], axis=1) tmp = gpd.pd.concat([df.reset_index(drop=True),zonal], axis=1)
tmp = tmp.set_geometry('geom', crs=2154) tmp = tmp.set_geometry('geom', crs=2154)
tmp.infeq_1200 = tmp.max_alti <= 1200 tmp['infeq_1200'] = tmp.max_alti <= 1200
PS = tmp[['site_code','infeq_1200']]\ PS = (tmp[['site_code','infeq_1200']]
.rename(columns={'site_code':'id_site'}) .rename(columns={'site_code':'id_site'},errors='ignore')
)
PS.to_sql('r_infeq_1200m',con,'ps',if_exists='append',index=False) PS.to_sql('r_infeq_1200m',con,'ps',if_exists='append',index=False)
# update_to_sql(PS,con,'r_infeq_1200m','ps','id_site') # update_to_sql(PS,con,'r_infeq_1200m','ps','id_site')
ps_all = gpd.read_postgis(''' ps_all = gpd.read_postgis('''
SELECT vp.*,s7.infeq_1200 SELECT vp.*,s7.infeq_1200
FROM ps."v_pelouseseches_all" vp FROM ps."v_pelouseseches_all" vp
LEFT JOIN ps.r_infeq_1200m s7 ON vp.site_code::text = s7.id_site::text LEFT JOIN ps.r_infeq_1200m s7 ON vp.site_code::text = s7.id_site::text
ORDER BY vp.site_code DESC NULLS LAST; ORDER BY vp.site_code DESC NULLS LAST;
''',con) ''', con)
isnot_1200 = ps_all.infeq_1200.isna() isnot_1200 = ps_all.infeq_1200.isna()
ps_all[~ps_all.infeq_1200]
update_psInf1200(ps_all[isnot_1200]) update_psInf1200(ps_all[isnot_1200])