init depot renew

This commit is contained in:
2026-09-22 11:58:57 +02:00
commit e9fa6a7309
205 changed files with 39721 additions and 0 deletions
+1
View File
@@ -0,0 +1 @@
#!/usr/bin/env python3
+263
View File
@@ -0,0 +1,263 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
import geopandas as gpd
from geoalchemy2 import Geometry
def _calc_recouvrmt(df1,df2,how='inner'):
'''
Calcule le recouvrement de df2 sur df1
pour chaque géométrie de df1:
Parameters
----------
df1 : GeoDataFrame.
df2 : GeoDataFrame.
'''
iddf1 = df1.columns[0]
iddf2 = df2.columns[0]
# Jointure spaciale
tmp = gpd.sjoin(
df1,
df2[['geom']],
predicate = 'intersects',
how = how)
tmp.dropna(subset=['index_right'],inplace=True)
tmp.index_right = tmp.index_right.astype(int)
tmp.reset_index(inplace=True)
tmp = tmp.join(
df2[['geom',iddf2]].rename(columns={'geom': 'right_geom'}),
on=['index_right'], how='left')
tmp2 = tmp[['index_right','right_geom',iddf2]].copy() \
.rename(columns={'right_geom': 'geom'}) \
.set_geometry('geom')
tmp1 = tmp[[iddf1,'geom']].copy() \
.set_geometry('geom')
if not tmp1.geom.values.is_valid.all():
tmp1.loc[~tmp1.geom.values.is_valid,'geom'] = tmp1.loc[~tmp1.geom.values.is_valid,'geom'].buffer(0)
if not tmp2.geom.values.is_valid.all():
tmp2.loc[~tmp2.geom.values.is_valid,'geom'] = tmp2.loc[~tmp2.geom.values.is_valid,'geom'].buffer(0)
tmp['perc_rcvmt'] = (tmp1.intersection(tmp2).area/tmp1.area)*100
return tmp[[iddf1,iddf2,'perc_rcvmt']]
def _cor_zh_area(con,tzh_code,typ,cover=False):
"""
@tzh : pd.Serie. Série de valeurs
correspondants à la colonne pr_zh.t_zh."code".
@typ : str. COM, DEP, ref_geo
"""
from math import ceil
table = 'cor_zh_area'
sqltzh = """
SELECT zh.id_zh, zh.geom FROM pr_zh.t_zh zh WHERE zh."code" in {tzh_code}
""".format(tzh_code=tuple(tzh_code))
tzh = gpd.read_postgis(sqltzh,con,crs=4326)
if tzh.crs.srs=='epsg:4326':
tzh.to_crs(2154,inplace=True)
if typ == 'COM':
cd2 = ' AND l."enable";'
else: cd2 = ';'
# sql = '''
# SELECT l.id_area,l.geom,zh.id_zh
# FROM ref_geo.l_areas l
# JOIN ref_geo.bib_areas_types bib USING (id_type),
# pr_zh.t_zh zh
# WHERE zh."code" in {tzh_code}
# AND bib.type_code='{typ}'
# AND ST_INTERSECTS( ST_SetSRID(l.geom,2154), ST_MakeValid(ST_Transform(ST_SetSRID(zh.geom,4326),2154)) )
# AND (l.id_area,zh.id_zh) NOT IN (SELECT id_area,id_zh FROM pr_zh.cor_zh_area)
# {cd2}
# '''.format(tzh_code=tuple(tzh_code),typ=typ,cd2=cd2)
# # larea = gpd.pd.read_sql_query(sql,con)
# larea = gpd.read_postgis(sql,con)
sqllarea = """
SELECT l.id_area, l.geom FROM ref_geo.l_areas l
JOIN ref_geo.bib_areas_types bib USING (id_type)
WHERE bib.type_code='{typ}' and l."enable"
""".format(typ=typ)
larea = gpd.read_postgis(sqllarea,con,crs=2154)
df = _calc_recouvrmt(larea,tzh).rename(columns={'perc_rcvmt':'cover'})
if cover:
df['cover'] = [ceil(x) for x in df.cover]
else :
df.drop(columns=['cover'],inplace=True)
if not df.empty:
df.to_sql(
name=table, con=con, schema='pr_zh',
if_exists='append', index=False
)
print('INSERT %i correspondances'%df.shape[0])
else:
sql = '''
SELECT l.id_area
FROM ref_geo.l_areas l
JOIN ref_geo.bib_areas_types bib USING (id_type)
WHERE bib.type_code='{typ}'
'''.format(typ=typ)
res = gpd.pd.read_sql_query(sql,con)
if not res.empty:
print('AUCUNE nouvelles correspondances identifiées')
else :
print('AUCUNE geometrie dans la table `ref_geo.l_areas` pour le `type_code` %s'%typ)
def _cor_zh_(con,tzh_code,typ):
"""
@tzh : pd.Serie. Série de valeurs
correspondants à la colonne pr_zh.t_zh."code".
@typ : str. [hydro,rb]
"""
typ = typ.lower()
table = 'cor_zh_%s'%typ
tab_typ = 't_hydro_area' if typ == 'hydro' else 't_river_basin'
id_typ = 'id_hydro' if typ == 'hydro' else 'id_rb'
sql = '''
SELECT h.{id_typ},zh.id_zh
FROM pr_zh.{tab_typ} h, pr_zh.t_zh zh
WHERE zh."code" in {tzh_code}
AND ST_INTERSECTS( ST_SetSRID(h.geom,4326),ST_MakeValid(ST_SetSRID(zh.geom,4326)))
AND (h.{id_typ},zh.id_zh) NOT IN (SELECT {id_typ},id_zh FROM pr_zh.{tab_to})
;'''.format(
tzh_code = tuple(tzh_code),
id_typ = id_typ,
tab_typ = tab_typ,
tab_to = table)
df = gpd.pd.read_sql_query(sql,con)
if not df.empty:
df.to_sql(
name=table, con=con, schema='pr_zh',
if_exists='append', index=False
)
print('INSERT %i correspondances'%df.shape[0])
else:
print('AUCUNE nouvelles correspondances identifiées')
def get_id_t_zh(con,code=None):
"""@code : str, list, Serie, Index. Code à 12 characters maximum de la zone humide.
"""
sql = "SELECT id_zh,code FROM pr_zh.t_zh" # "SELECT id_zh,zh_uuid,code FROM pr_zh.t_zh"
if isinstance(code,str):
sql += " WHERE code='%s'"%code
elif isinstance(code,list) or isinstance(code,gpd.pd.Series) or isinstance(code,gpd.pd.Index):
sql += " WHERE code IN {lst_code}".format(lst_code=str(tuple(code)))
return gpd.pd.read_sql_query(sql,con)
def to_tzh(df,con):
table = 't_zh'
if isinstance(df,gpd.GeoDataFrame):
epsg = df.crs.srs.split(':')[1]
df = df.to_wkt()
df.to_sql(
name=table, con=con, schema='pr_zh',
if_exists='append', index=False,
dtype={
'geom': Geometry(srid=epsg)
# 'id_lim_list':uuid.SafeUUID
}
)
if isinstance(df,gpd.pd.DataFrame):
df.geom = gpd.GeoSeries.from_wkt(df.geom)
df = df.set_geometry('geom',crs=epsg)
_cor_zh_area(con,tzh_code=df.code,typ='ZNIEFF1')
print('INSERT cor_zh_area ZNIEFF1 OK !')
_cor_zh_area(con,tzh_code=df.code,typ='ZNIEFF2')
print('INSERT cor_zh_area ZNIEFF2 OK !')
_cor_zh_area(con,tzh_code=df.code,typ='DEP')
print('INSERT cor_zh_area DEP OK !')
_cor_zh_area(con,tzh_code=df.code,typ='COM',cover=True)
print('INSERT cor_zh_area COM OK !')
_cor_zh_area(con,tzh_code=df.code,typ='ZPS')
print('INSERT cor_zh_area ZPS OK !')
_cor_zh_area(con,tzh_code=df.code,typ='SIC')
print('INSERT cor_zh_area SIC OK !')
_cor_zh_area(con,tzh_code=df.code,typ='SRAM')
print('INSERT cor_zh_area SRAM OK !')
_cor_zh_area(con,tzh_code=df.code,typ='ZICO')
print('INSERT cor_zh_area ZICO OK !')
_cor_zh_(con,tzh_code=df.code,typ='hydro')
print('INSERT cor_zh_hydro OK !')
_cor_zh_(con,tzh_code=df.code,typ='rb')
print('INSERT cor_zh_rb OK !')
def to_cor_(con,df,table):
df.to_sql(
name=table,con=con,schema='pr_zh',if_exists='append',index=False
)
def to_t_(con,df,table):
res = gpd.pd.merge(df,get_id_t_zh(con,df.SITE_COD),left_on='SITE_COD',right_on='code')\
.drop(columns=['SITE_COD','code'])
res.to_sql(
name=table,con=con,schema='pr_zh',if_exists='append',index=False
)
def add_bib_cb(df,con,humidity=None,is_ch=None):
"""
@df : pd.Dataframe. Tableau à 3 colonnes |lb_code|humidity|is_ch|.
@con : engine. Connexion PostgreSQL à la base de données cible.
@humidity : str. Caractère d'humidité a attribuer par défault
à chaque lb_code du tableau où la colonne est Null.
Défaut : None (désactivé)
Liste : P (Potentiellement), H (Humide).
@is_ch : bool. Caractère de présence/absence de carto d'habitat
à attribuer par défault à chaque lb_code du tableau où la colonne est Null.
Défaut : None (désactivé)
Liste : True (existe), False (n'existe pas).
"""
table = 'bib_cb'
if humidity is not None:
df.loc[df.humidity.isna(),'humidity'] = humidity.upper()
if is_ch is not None:
df.loc[df.is_ch.isna(),'is_ch'] = is_ch
bib_cb = gpd.pd.read_sql_table(table,con,'pr_zh')
df = df[~df.lb_code.isin(bib_cb.lb_code)]
if df.empty:
print('NO news data to insert !')
else:
df.to_sql(
name=table,con=con,schema='pr_zh',if_exists='append',index=False
)
print('INSERT %s news rows !'%df.shape[0])
if __name__ == '__main__':
from ..MEDWET2Geonat import get_t_zh
zh,_cor_lim_list = get_t_zh()
+222
View File
@@ -0,0 +1,222 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from owslib.wfs import WebFeatureService
import geopandas as gpd
from geoalchemy2 import Geometry
from sqlalchemy import dialects
def get_wfs(url, layer, bbox=None):
from geopandas import read_file
from requests import Request
wfs = WebFeatureService(url=url)
item = dict(wfs.items())[layer]
crs = item.crsOptions[0].getcode()
params = dict(service='WFS', version=wfs.version, request='GetFeature',
typeName=layer)
q = Request('GET', url, params=params).prepare().url
data = read_file(q, bbox=bbox)
data.set_crs(crs=crs, inplace=True)
if crs != 'EPSG:2154':
data.to_crs(epsg=2154, inplace=True)
return data
def list_layer(url):
wfs = WebFeatureService(url=url)
lst = list(wfs.contents)
return lst
def Polygons_to_MultiPolygon(df,geom_col=None):
from shapely.geometry import MultiPolygon
from pandas import concat
if not geom_col:
geom_col = df.geometry.name
df = df.copy()
multi = df.loc[df.geom_type=='MultiPolygon'].copy()
poly = df.loc[df.geom_type=='Polygon'].copy()
poly[geom_col] = [MultiPolygon([geom]) for geom in df.loc[df.geom_type=='Polygon',geom_col] ]
df = concat([multi,poly])
df.sort_index(inplace=True)
return df
def ref_geo(type_code,con):
sql = """
SELECT l.* FROM ref_geo.l_areas l
JOIN ref_geo.bib_areas_types bib USING (id_type)
WHERE bib.type_code='{typ}'
""".format(typ=type_code)
return gpd.read_postgis(sql,con)
def to_lareas(df,dic,layer,con,dtypes={}):
id_type = gpd.pd.read_sql_query("""
SELECT id_type FROM ref_geo.bib_areas_types WHERE type_code='%s'
"""%layer,con).id_type.values[0]
ref = ref_geo(layer,con)
df.rename(columns=dic, inplace=True)
df = Polygons_to_MultiPolygon(
df[~df.area_name.isin(ref.area_name)]
)
df.rename_geometry('geom', inplace=True)
del_col = df.columns[~df.columns.isin(['geom',*[*set(dic.values())]])]
df.drop(columns=del_col, inplace=True)
df['id_type'] = id_type
df['geojson_4326'] = df.to_crs(4326).geom.__geo_interface__['features']
df['geojson_4326'] = [x['geometry'] for x in df['geojson_4326']]
df['centroid'] = 'SRID=2154;'+df.geom.centroid.to_wkt()
df['enable'] = True
if df.empty:
print('AUCUN nouveaux zonages identifiés')
else:
df.to_postgis(
name='l_areas',
con=con,
schema='ref_geo',
if_exists='append',
index=False,
index_label=None,
chunksize=None,
dtype={
'centroid': Geometry(geometry_type='POINT',srid=2154),
**dtypes
},
)
print('INSERT %i zones'%df.shape[0])
def update_larea(con,layer,df,cols_updt=[],on='area_name'):
from pycen import update_to_sql
table = 'l_areas'
# idtyp = tuple(df.id_type.unique())
idtyp = gpd.pd.read_sql_query("""
SELECT id_type FROM ref_geo.bib_areas_types WHERE type_code='%s'
"""%layer,con).id_type.values[0]
# if len(idtyp) > 1:
# where = 'in %s'%tuple(df.id_type.unique())
# else :
where = '=%i'%idtyp
sql = 'SELECT id_area, %s FROM ref_geo.l_areas WHERE id_type %s;'%(on,where)
larea = gpd.pd.read_sql_query(sql,con)
to_updt = df.merge(larea, on=on)
update_to_sql(
df=to_updt[['id_area',*cols_updt]],
con=con,
table_name=table,
schema_name='ref_geo',
key_name='id_area',
)
carmen = 'https://ws.carmencarto.fr/WFS/119/fxx_inpn?'
dic_layer = {
# ref_geo.bib_areas_types.type_code : layer_name from list_layer
'ZNIEFF2':'Znieff2',
'ZNIEFF1':'Znieff1',
'APB':'Arretes_de_protection_de_biotope',
'RNN':'Reserves_naturelles_nationales',
'RNR':'Reserves_naturelles_regionales',
'ZPS':'Zones_de_protection_speciale',
'SIC':'Sites_d_importance_communautaire',
'ZICO':'ZICO',
'RNCFS':'Reserves_nationales_de_chasse_et_faune_sauvage',
'RIPN':'Reserves_Integrales_de_Parcs_Nationaux', # NO zone in 74
'SCEN':'Terrains_acquis_des_Conservatoires_des_espaces_naturels', # Pas fait ;
# 'SCL':'',
# 'PNM':'',
'PNR':'Parcs_naturels_regionaux',
'RBIOL':'Reserves_biologiques',
'RBIOS':'Reserves_de_la_biosphere', # NO zone in 74
# 'RNC':'',
'SRAM':'Sites_Ramsar',
# 'AA':'',
# 'ZSC':'',
# 'PSIC':'',
# 'PEC':'',
# ...
}
if __name__ == "__main__":
from sqlalchemy import create_engine # pour lecture de la bd Géonature
from sqlalchemy.engine import URL
usr = 'XXXXX'
pdw = 'XXXXX'
host= 'XXXXX'
bd = 'XXXXX'
eng = URL.create('postgresql+psycopg2',username=usr,password=pdw,host=host,database=bd)
conn = create_engine(eng)
# from pycen import con_gn as conn
ref = ref_geo('DEP',conn)
ter = ref[ref.area_code=='74']
layer = 'Znieff1'
# layer_name = dic_layer[layer]
zon = get_wfs(carmen,layer,bbox=ter.unary_union)
zon = zon[zon.intersects(ter.unary_union)]
dic_cols = {
'ID_MNHN':'area_code',
'NOM':'area_name',
'URL':'source',
}
dtyp = {}
if layer in ['ZPS','SIC']:
dic_cols = {**dic_cols,**{'SITECODE':'area_code','SITENAME':'area_name',}}
elif layer == 'RNN':
dic_cols = {**dic_cols,**{'URL_FICHE':'source','NOM_SITE':'area_name',}}
elif layer == 'ZICO':
dic_cols = {**dic_cols,**{'ID_SPN':'area_code'}}
elif layer == 'SCEN':
zon.NOM = zon.NOM.str.title()
elif layer == 'RBIOL':
dic_cols = {
**dic_cols,
**{'URL_FICHE':'source','NOM_SITE':'area_name',
'comment':'comment','additional_data':'additional_data',}
}
zon['comment'] = 'Réserves biologiques ' + zon.CODE_R_ENP.replace(
{'I': 'intégrale','D' : 'dirigée'}
)
zon['additional_data'] = [
str({ zon.CODE_R_ENP.name : x }).replace("\'",'\"')
for x in zon.CODE_R_ENP
]
dtyp = {'additional_data':dialects.postgresql.JSONB}
# IF UPDATE DATA
# update_larea(conn,layer,zon.rename(columns=dic_cols),['comment'],on='area_code')
to_lareas(
df=zon,
dic=dic_cols,
layer=layer,
con=conn,
dtypes=dtyp
)
+37
View File
@@ -0,0 +1,37 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
import geopandas as gpd
from pycen import update_to_sql, con_gn, con as con_aza
def get_sdr_eau(con=con_aza):
return gpd.read_postgis('SELECT * FROM ref_territoire.sdr_ressources_en_eau', con)
def get_zh_to_update(con=con_gn):
return gpd.read_postgis("SELECT id_zh,code,ST_TRANSFORM(geom,2154) geom FROM pr_zh.t_zh WHERE code ilike '38CENIS%%'", con)
def change_code(zh,sdr,con=con_gn):
res = zh.sjoin(sdr[['code_tr','geom']]).drop(columns='index_right')
code_max = gpd.pd.read_sql(
"""SELECT MAX(SUBSTRING(code,5,4)::int) val_max, SUBSTRING(code,3,2) code_tr
FROM pr_zh.t_zh WHERE code ilike ANY(ARRAY['%%{}%%'])
GROUP BY SUBSTRING(code,3,2)""".format(
"%%','%%".join(res.code_tr.unique().tolist())
),con_gn
)
for i,r in code_max.iterrows():
res.loc[res.code_tr==r.code_tr,'code_new'] = ['38'+row.code_tr+str(r.val_max+i+1).zfill(4) for i,row in res.iterrows()]
update_to_sql(
res[['id_zh','code_new']].rename(columns={'code_new':'code'}),
con_gn,
't_zh',
'pr_zh',
'id_zh'
)
if __name__ == "__main__":
sdr = get_sdr_eau(con=con_aza)
zh = get_zh_to_update(con=con_gn)
change_code(zh,sdr,con=con_gn)
+31
View File
@@ -0,0 +1,31 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
import os
import pandas as pd
import geopandas as gpd
from geoalchemy2 import Geometry
def move2filedir():
'''
Orientation vers le répertoire où est localiser le script zh2gn.py
'''
dir_path = os.path.dirname(os.path.realpath(__file__))
os.chdir(dir_path)
# print(os.getcwd())
if __name__ == '__main__':
# move2filedir()
os.chdir('/home/colas/Documents/13_CEN74/medwet2gn_ZH/tools')
PATH = '../template/'
FILE_DATA = 'BDD_ZH_CCPR.xlsx'
GEO_DATA = 'ZH_CCPR_finalisées/ZH_asters_finalisées.shp'
data = pd.read_excel(PATH+FILE_DATA,'formulaire_type')
geo = gpd.read_file(PATH+GEO_DATA)