Compare commits
No commits in common. "51d7ab1bd4079c214e879c1d96184b44f6700319" and "ba5ef578d03d7dc79da39b6853ad9c2edaa5edc0" have entirely different histories.
51d7ab1bd4
...
ba5ef578d0
@ -1,6 +1,4 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-.
|
||||
|
||||
from csv import list_dialects
|
||||
import geopandas as gpd
|
||||
from uuid import uuid4
|
||||
import json,urllib.request
|
||||
@ -12,38 +10,14 @@ dict_columns = {
|
||||
'code_zh': 'code',
|
||||
'nom_zh': 'main_name',
|
||||
'date_visite':'create_date',
|
||||
'remark_activity': 'global_remark_activity'
|
||||
}
|
||||
|
||||
dict_type_milieux = {
|
||||
'FU':'Fourré humide',
|
||||
'BFH':'Boisement feuillu humide',
|
||||
'BCH':'Boisement de conifères humide',
|
||||
'AL':'Végétation herbacée pionnière des alluvions',
|
||||
'GH':'Communauté de grands hélophytes',
|
||||
'RB':'Communauté de petits hélophytes',
|
||||
'MC':'Magnocariçaie',
|
||||
'BM':'Bas-marais et marais de transition',
|
||||
'MG':'Mégaphorbiaie',
|
||||
'AQ':'Végétation aquatique',
|
||||
'FO':'Végétation fontinale (sources, mouillères)',
|
||||
'EC':"Végétation amphibie des bordure d'eaux courantes",
|
||||
'PH':'Prairie humide (et pelouse humide)',
|
||||
}
|
||||
|
||||
|
||||
def normalize_connexion(con):
|
||||
if gpd.pd.__version__>'2.1.2':
|
||||
return con.raw_connection()
|
||||
else:
|
||||
return con
|
||||
|
||||
|
||||
def get_cor_impact_types(con):
|
||||
res = gpd.pd.read_sql_table('cor_impact_types',con,'pr_zh')
|
||||
return dict(zip(res.id_impact,res.id_cor_impact_types))
|
||||
|
||||
|
||||
def get_nomenclature_id(con,cd,typ):
|
||||
"""
|
||||
Get the id_nomenclature for a given cd and typ.
|
||||
@ -71,7 +45,7 @@ def get_nomenclature_id(con,cd,typ):
|
||||
return res.one()[0] if res else None
|
||||
|
||||
|
||||
def _cor_zh_area(tzh_code,con,typ,cover=False):
|
||||
def _cor_zh_area(tzh_code,typ,cover=False):
|
||||
"""
|
||||
@tzh : pd.Serie. Série de valeurs
|
||||
correspondants à la colonne pr_zh.t_zh."code".
|
||||
@ -84,7 +58,7 @@ def _cor_zh_area(tzh_code,con,typ,cover=False):
|
||||
sqltzh = """
|
||||
SELECT zh.id_zh, zh.geom FROM pr_zh.t_zh zh WHERE zh."code" in {tzh_code}
|
||||
""".format(tzh_code=tuple(tzh_code))
|
||||
tzh = gpd.read_postgis(sqltzh,con,crs=4326)
|
||||
tzh = gpd.read_postgis(sqltzh,con_gn,crs=4326)
|
||||
|
||||
if tzh.crs.srs.lower()=='epsg:4326':
|
||||
tzh.to_crs(2154,inplace=True)
|
||||
@ -94,7 +68,7 @@ def _cor_zh_area(tzh_code,con,typ,cover=False):
|
||||
JOIN ref_geo.bib_areas_types bib USING (id_type)
|
||||
WHERE bib.type_code='{typ}' and l."enable"
|
||||
""".format(typ=typ)
|
||||
larea = gpd.read_postgis(sqllarea,con,crs=2154)
|
||||
larea = gpd.read_postgis(sqllarea,con_gn,crs=2154)
|
||||
|
||||
df = _calc_recouvrmt(larea,tzh).rename(columns={'perc_rcvmt':'cover'})
|
||||
|
||||
@ -106,7 +80,7 @@ def _cor_zh_area(tzh_code,con,typ,cover=False):
|
||||
# return df
|
||||
if not df.empty:
|
||||
df.to_sql(
|
||||
name=table, con=con, schema='pr_zh',
|
||||
name=table, con=con_gn, schema='pr_zh',
|
||||
if_exists='append', index=False
|
||||
)
|
||||
print('INSERT %i correspondances'%df.shape[0])
|
||||
@ -159,7 +133,32 @@ def _calc_recouvrmt(df1,df2,how='inner'):
|
||||
return tmp[[iddf1,iddf2,'perc_rcvmt']]
|
||||
|
||||
|
||||
def _cor_zh_(tzh_code,con,typ):
|
||||
def _cor_zh_hydro(tzh_code):
|
||||
"""
|
||||
@tzh : pd.Serie. Série de valeurs
|
||||
correspondants à la colonne pr_zh.t_zh."code".
|
||||
"""
|
||||
table = 'cor_zh_hydro'
|
||||
sql = '''
|
||||
SELECT h.id_hydro,zh.id_zh
|
||||
FROM pr_zh.t_hydro_area h, pr_zh.t_zh zh
|
||||
WHERE zh."code" in {tzh_code}
|
||||
AND ST_INTERSECTS( ST_SetSRID(h.geom,4326),ST_MakeValid(ST_SetSRID(zh.geom,4326)))
|
||||
AND (h.id_hydro,zh.id_zh) NOT IN (SELECT id_hydro,id_zh FROM pr_zh.cor_zh_hydro)
|
||||
'''.format(tzh_code=tuple(tzh_code))
|
||||
df = gpd.pd.read_sql_query(sql,con_gn)
|
||||
|
||||
if not df.empty:
|
||||
df.to_sql(
|
||||
name=table, con=con_gn, schema='pr_zh',
|
||||
if_exists='append', index=False
|
||||
)
|
||||
print('INSERT %i correspondances'%df.shape[0])
|
||||
else:
|
||||
print('AUCUNE nouvelles correspondances identifiées')
|
||||
|
||||
|
||||
def _cor_zh_(tzh_code,typ):
|
||||
"""
|
||||
@tzh : pd.Serie. Série de valeurs
|
||||
correspondants à la colonne pr_zh.t_zh."code".
|
||||
@ -180,11 +179,11 @@ def _cor_zh_(tzh_code,con,typ):
|
||||
id_typ = id_typ,
|
||||
tab_typ = tab_typ,
|
||||
tab_to = table)
|
||||
df = gpd.pd.read_sql_query(sql,con)
|
||||
df = gpd.pd.read_sql_query(sql,con_gn)
|
||||
|
||||
if not df.empty:
|
||||
df.to_sql(
|
||||
name=table, con=con, schema='pr_zh',
|
||||
name=table, con=con_gn, schema='pr_zh',
|
||||
if_exists='append', index=False
|
||||
)
|
||||
print('INSERT %i correspondances'%df.shape[0])
|
||||
@ -237,12 +236,13 @@ def to_tzh(df,con,lst_area):
|
||||
res_df = gpd.pd.DataFrame(res)
|
||||
for area in lst_area:
|
||||
cov = True if area == 'COM' else False
|
||||
_cor_zh_area(tzh_code=res_df.code,con=con,typ=area,cover=cov)
|
||||
_cor_zh_area(tzh_code=res_df.code,typ=area,cover=cov)
|
||||
print('INSERT cor_zh_area %s OK !'%area)
|
||||
|
||||
_cor_zh_(tzh_code=res_df.code,con=con,typ='hydro')
|
||||
# _cor_zh_hydro(tzh_code=res_df.code)
|
||||
_cor_zh_(tzh_code=res_df.code,typ='hydro')
|
||||
print('INSERT cor_zh_hydro OK !')
|
||||
_cor_zh_(tzh_code=res_df.code,con=con,typ='rb')
|
||||
_cor_zh_(tzh_code=res_df.code,typ='rb')
|
||||
print('INSERT cor_zh_rb OK !')
|
||||
|
||||
print("INSERT data TO t_zh OK !")
|
||||
@ -313,7 +313,6 @@ def to_cor_zh_cb(id_zh,df,con):
|
||||
-------
|
||||
None
|
||||
"""
|
||||
|
||||
cor_zh_cb = (df
|
||||
.merge(id_zh,how='left',on='code')
|
||||
.set_index('id_zh')
|
||||
@ -338,13 +337,11 @@ def to_cor_zh_cb(id_zh,df,con):
|
||||
finally:
|
||||
print("INSERT habitats to cor_zh_cb OK !")
|
||||
|
||||
|
||||
def check_cd_nomenclature_impact(cd):
|
||||
dizaine = range(0,100,10)
|
||||
_cd = [x+'.0' if float(x) not in dizaine and not x.endswith('.0') else x for x in cd]
|
||||
return _cd
|
||||
|
||||
|
||||
def to_t_activity(id_zh,actv,con):
|
||||
_activ = (actv
|
||||
.merge(id_zh,how='left',on='code')
|
||||
@ -373,10 +370,7 @@ def to_t_activity(id_zh,actv,con):
|
||||
.reset_index()
|
||||
)
|
||||
impact_list['_cd_impact'] = check_cd_nomenclature_impact(impact_list.cd_impact)
|
||||
impact_list['id_cor_impact_types'] = [get_nomenclature_id(con,x,'IMPACTS') for x in impact_list._cd_impact]
|
||||
|
||||
cor_imp_typ = get_cor_impact_types(con)
|
||||
impact_list.replace({'id_cor_impact_types':cor_imp_typ}, inplace=True)
|
||||
impact_list['id_impact'] = [get_nomenclature_id(con,x,'IMPACTS') for x in impact_list._cd_impact]
|
||||
|
||||
t_activ.drop(['cd_activite_humaine','localisation','impact'],axis=1,inplace=True)
|
||||
impact_list.drop(['cd_impact','_cd_impact'],axis=1,inplace=True)
|
||||
@ -401,7 +395,7 @@ def to_t_activity(id_zh,actv,con):
|
||||
name='cor_impact_list',
|
||||
con=con,
|
||||
schema='pr_zh',
|
||||
index=False,
|
||||
index=True,
|
||||
if_exists='append',
|
||||
method='multi'
|
||||
)
|
||||
@ -575,9 +569,7 @@ def filter_zh(code,con):
|
||||
res = gpd.pd.read_sql(sql,con)
|
||||
return res
|
||||
|
||||
def insert_zh_fromapi(
|
||||
url,con,dep_filter,orga,lst_area=['DEP','COM'],
|
||||
prefix_hab_rq='',code_rename={}):
|
||||
def insert_zh_fromapi(url,con,dep_filter,orga,lst_area=['DEP','COM'],prefix_hab_rq=''):
|
||||
"""
|
||||
Insert data from API into geonature database
|
||||
|
||||
@ -591,59 +583,21 @@ def insert_zh_fromapi(
|
||||
filter code_zh by this department code
|
||||
prefix_hab_rq : str
|
||||
prefix to add to remark_pres field for habitats not in geonature database
|
||||
code_rename : dict
|
||||
Dictionnary of zh code
|
||||
|
||||
|
||||
Returns
|
||||
-------
|
||||
None
|
||||
"""
|
||||
|
||||
api = gpd.read_file(url, engine='fiona')
|
||||
api = gpd.read_file(url)
|
||||
if api.empty:
|
||||
print("Aucune zone humide trouvée")
|
||||
print("Arrêt du script !")
|
||||
print("Aucun zone humide trouvée")
|
||||
return None
|
||||
|
||||
api.rename(columns=dict_columns, inplace=True)
|
||||
if code_rename :
|
||||
api.replace({'code':code_rename},inplace=True)
|
||||
|
||||
zh_filter = filter_zh(
|
||||
tuple(api.code.dropna()),
|
||||
con
|
||||
)
|
||||
|
||||
nzh_tobdd = api[
|
||||
(api.action=='Créer')&
|
||||
(~api.code.isin(zh_filter.code.tolist()))&
|
||||
(~api.cd_typo_sdage.isna())
|
||||
].shape[0]
|
||||
print("L'api contient %s zones .."%api.shape[0])
|
||||
print("%s zones sont taguée `Problème` .."%api[api.action=='Problème'].shape[0])
|
||||
print("%s zones dont la typologie SDAGE est inconnue et sont écartées"%api[api.cd_typo_sdage.isna()].shape[0])
|
||||
print("%s zones taguée `Créer` existent déjà .."%zh_filter.shape[0])
|
||||
print("%s zones taguée `Créer` sont à intégrer .."%nzh_tobdd)
|
||||
|
||||
if nzh_tobdd == 0 :
|
||||
print("\nAucune nouvelles zones humides sont à intégrée ..")
|
||||
print("Arrêt du script !")
|
||||
return None
|
||||
|
||||
Q = input('Souhaitez-vous continuer ? (Y/n) : ')
|
||||
if Q.lower() != 'y':
|
||||
print('Arrêt du code ..')
|
||||
return None
|
||||
|
||||
df = (api[
|
||||
(api.action=='Créer')&
|
||||
(~api.code.isin(zh_filter.code.tolist()))&
|
||||
(~api.cd_typo_sdage.isna())
|
||||
].rename_geometry('geom')
|
||||
df = (api[api.action=='Créer']
|
||||
.rename(columns=dict_columns)
|
||||
.rename_geometry('geom')
|
||||
# .merge(load_missing_propertie(url,'cd_nomenclature_delimitation',dep_filter), on='pk')
|
||||
)
|
||||
|
||||
df.observateur = normalize_observers(df.observateur)
|
||||
users, id_org = insert_users_missing(df.observateur,orga,con)
|
||||
df['id_org'] = id_org
|
||||
@ -652,12 +606,12 @@ def insert_zh_fromapi(
|
||||
.rename(columns={'ids_observers':'id_role'})
|
||||
)
|
||||
|
||||
|
||||
df['zh_uuid'] = [uuid4() for _ in range(len(df))]
|
||||
df['id_lim_list'] = [uuid4() for _ in range(len(df))]
|
||||
df['id_sdage'] = [get_nomenclature_id(con,x,'SDAGE') for x in df.cd_typo_sdage]
|
||||
df['create_author'] = df.id_role.copy()
|
||||
df['update_author'] = df.id_role.copy()
|
||||
df['area'] = df.to_crs(2154).area/10000
|
||||
|
||||
cd_hab = (df
|
||||
.set_index('code')
|
||||
@ -668,33 +622,16 @@ def insert_zh_fromapi(
|
||||
hab_zh,hab_notzh = filter_habitat(cd_hab,con)
|
||||
_df = (df
|
||||
.drop('habitat_corine_biotope',axis=1)
|
||||
.merge(hab_zh,how='left',right_index=True,left_on='code')
|
||||
.sort_values('code'))
|
||||
|
||||
# Si pas de champs remark_pres dans le tableau
|
||||
if 'remark_pres' not in _df.columns:
|
||||
_df = _df.merge(
|
||||
prefix_hab_rq + hab_notzh.rename('remark_pres'),
|
||||
how='left',right_index=True,left_on='code'
|
||||
)
|
||||
else :
|
||||
_df.remark_pres = (_df.remark_pres
|
||||
.str.split('_',expand=True)[0]
|
||||
.replace(dict_type_milieux)
|
||||
)
|
||||
_df.set_index('code',inplace=True)
|
||||
_df.loc[_df.index.isin(hab_notzh.index),'remark_pres'] = (
|
||||
_df[_df.index.isin(hab_notzh.index.tolist())]
|
||||
.remark_pres
|
||||
+ '\n' + prefix_hab_rq + hab_notzh
|
||||
)
|
||||
_df.reset_index(drop=False,inplace=True)
|
||||
.merge(hab_zh,how='left',right_index=True,left_on='code'))
|
||||
_df = _df.merge(
|
||||
prefix_hab_rq + hab_notzh.rename('remark_pres'),
|
||||
how='left',right_index=True,left_on='code'
|
||||
)
|
||||
|
||||
c = _df[_df.action=="Créer"].copy()
|
||||
u = _df[_df.action=="Modifier"].copy()
|
||||
|
||||
if not c.empty:
|
||||
# retrun c
|
||||
id_zh = to_tzh(c,con,lst_area)
|
||||
to_cor_zh_cb(id_zh,c,con)
|
||||
to_cor_lim_list(c[['id_lim_list','cd_nomenclature_delimitation']],con)
|
||||
@ -740,34 +677,24 @@ if __name__ == "__main__":
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.engine import URL
|
||||
# Parametres bdd
|
||||
user = 'xxxxxxx'
|
||||
pwd = "xxxxxxx"
|
||||
adr = 'xxxxxxx'
|
||||
base = 'xxxxxxx'
|
||||
user = 'geonatadmin'
|
||||
pwd = "prep!!Ge0naT38*aDm1n"
|
||||
adr = 'localhost'
|
||||
base = 'geonature2db'
|
||||
url = URL.create("postgresql+psycopg2", username=user, password=pwd, host=adr, database=base)
|
||||
con_gn = create_engine(url)
|
||||
|
||||
# from pycen import con_gn
|
||||
# Numéro de département permettant d'identifier les zones humides concernées par le territoire
|
||||
# ['38', '05'], default : 38
|
||||
dep_filter = '05'
|
||||
# Préfixe ajouté dans le champs remark_pres lorsque des habitats décrits ne sont pas des habitats dit "humides" dans la listes des habitats humides définis en bdd (table bib_cb)
|
||||
dep_filter = '38'
|
||||
# Préfixe ajouté dans le champs remark_pres lorsque des habitats décrits ne sont pas des habitats dit "humides"
|
||||
prefix_hab_rmk = 'Autre(s) habitat(s) décrit(s) :\n'
|
||||
# [Nom de l'organisme, Abbreviation]
|
||||
organisme = ['Parc national des Écrins','PNE']
|
||||
|
||||
# Liste des type_area à intersecter pour remplir la table cor_zh_area
|
||||
listAreaCover = ['DEP','COM','APB','ZPS','SIC','ZNIEFF1','ZNIEFF2']
|
||||
listAreaCover = ['DEP','COM','APB','ZPS','SIC','ZNIEFF1','ZNIEFF2','ENS_ZI']
|
||||
|
||||
# Code zh de l'api à renomer si souhaité. Défaut = {}
|
||||
code_rename = {}
|
||||
|
||||
api = 'https://geonature.ecrins-parcnational.fr/api/exports/api/21?departement=%s'%dep_filter
|
||||
res = insert_zh_fromapi(
|
||||
url=api,
|
||||
con=con_gn,
|
||||
dep_filter=dep_filter,
|
||||
orga=organisme,
|
||||
lst_area=listAreaCover,
|
||||
prefix_hab_rq=prefix_hab_rmk,
|
||||
code_rename=code_rename
|
||||
)
|
||||
insert_zh_fromapi(api,con_gn,dep_filter,organisme,lst_area=listAreaCover,prefix_hab_rq=prefix_hab_rmk)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user