commit ca58cfaccb048885bc303d725b6bc0c3acdccb4c Author: Colas Geier Date: Fri Oct 8 15:14:47 2021 +0200 Initial commit diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..f362416 --- /dev/null +++ b/__init__.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- + +from .zh import * +from .bilan import * +from .update import update +from .pers import pers +from .sites import sites +from .ps import * +from .ref import * +from .params import con + + + +__version__ = "1.0.2" +__author__ = "Colas Geier" +__email__ = "colas.geier@cen-isere.org" +__all__ = [] +__license__ = "GPL" +__spec__ = 'Outils de récupération de données de zones humide de la base de données MEDWET' +__changes__ = { + "1.0.1" : "Création de la librairie, extraction des données zones humides et sites", + "1.0.2" : """Suppression des class et répartition des fonctions par sous-dossier pour faciliter l'appel. + Ajout de la récupération de données des pelouses sèches""" +} diff --git a/__pycache__/__init__.cpython-38.pyc b/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..bea6a64 Binary files /dev/null and b/__pycache__/__init__.cpython-38.pyc differ diff --git a/__pycache__/bilan.cpython-38.pyc b/__pycache__/bilan.cpython-38.pyc new file mode 100644 index 0000000..6ce295c Binary files /dev/null and b/__pycache__/bilan.cpython-38.pyc differ diff --git a/__pycache__/params.cpython-38.pyc b/__pycache__/params.cpython-38.pyc new file mode 100644 index 0000000..da5514e Binary files /dev/null and b/__pycache__/params.cpython-38.pyc differ diff --git a/__pycache__/ps.cpython-38.pyc b/__pycache__/ps.cpython-38.pyc new file mode 100644 index 0000000..d3556b7 Binary files /dev/null and b/__pycache__/ps.cpython-38.pyc differ diff --git a/__pycache__/tools.cpython-38.pyc b/__pycache__/tools.cpython-38.pyc new file mode 100644 index 0000000..77dda49 Binary files /dev/null and b/__pycache__/tools.cpython-38.pyc differ diff --git a/__pycache__/update.cpython-38.pyc b/__pycache__/update.cpython-38.pyc new file mode 100644 index 0000000..249d74a Binary files /dev/null and b/__pycache__/update.cpython-38.pyc differ diff --git a/__pycache__/zh.cpython-38.pyc b/__pycache__/zh.cpython-38.pyc new file mode 100644 index 0000000..3dd2cf1 Binary files /dev/null and b/__pycache__/zh.cpython-38.pyc differ diff --git a/bilan.py b/bilan.py new file mode 100644 index 0000000..e6599d1 --- /dev/null +++ b/bilan.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- +#Nom : : get_zh_cen.py +#Description : +#Copyright : 2021, CEN38 +#Auteur : Colas Geier +#Version : 1.0 + +from .zh import * +from .sites import sites + +##################################### +### Bilan ### +##################################### +def get_bilan(code_site=None, nom_site=None): + ''' + :sites: list,str. Nom de code du site de la zh. + ''' + SITES = sites + ZH = zh() + info = SITES.get_sitesInfos(ids=code_site, nom=nom_site) + CB = ZH.get_habitat(id_site=code_site, nom_site=nom_site) + delim = ZH.get_delim(id_site=code_site, nom_site=nom_site) + desc = ZH.get_usageprocess(id_site=code_site, nom_site=nom_site) + rghyd = ZH.get_regHydro(id_site=code_site, nom_site=nom_site) + subm = ZH.get_sub(id_site=code_site, nom_site=nom_site) + conn = ZH.get_connex(id_site=code_site, nom_site=nom_site) + fct = ZH.get_fct(id_site=code_site, nom_site=nom_site) + evall = SITES.get_sitesGeom(id_site=code_site, nom_site=nom_site).drop(columns=['geom']) + sub_con = pd.merge(subm, conn, how='outer', on=['id', 'id_site', 'date']) \ + .rename(columns={'description': 'desc_connex'}) + fctmt = { + 'entree_eau': rghyd[rghyd.in_out == 'entree'].drop(columns=['in_out'],errors='ignore'), + 'sortie_eau': rghyd[rghyd.in_out == 'sortie'].drop(columns=['in_out'],errors='ignore'), + 'sub_connex': sub_con, + } + lst_df = { + 'infos':info, + 'corine_biotope': CB, + 'delimitation': delim, + 'description': desc, + 'fonctionnement': fctmt, + 'fonction': fct, + 'evaluation': evall} + for key in lst_df: + if isinstance(lst_df[key], pd.DataFrame): lst_df[key].name = key + if isinstance(lst_df[key], dict): + for d in lst_df[key]: + lst_df[key][d].name = d + lst_df[key]['title'] = key + return lst_df + +def write_bilan(df, output): + ''' + :df: dict. Dictionnaire de DataFrame. + Ecriture d'un feuillet par élément du dictionnaire. + Le nom du DataFrame est le titre du feuillet. + output: str. chemin_vers_mon_fichier/mon_fichier.xlsx + ''' + # Ecriture des données + with pd.ExcelWriter(output) as writer: + for d in df: + DF = df[d] + if isinstance(DF, pd.DataFrame): + DF.to_excel(writer,sheet_name=DF.name,startrow=1 , startcol=0, index=False, header=DF.columns) + ws = writer.book.active + writer.sheets[DF.name].cell(1,1,value=DF.name) + writer.save() + elif isinstance(DF, dict): + for i,d in enumerate(DF): + if d == 'title': continue + if i == 0: + row = 1 + col = 0 + else: + col = DF[d].shape[1] + col + 3 + DF[d].to_excel(writer,sheet_name=DF['title'],startrow=row , startcol=col, index=False) + ws = writer.book.active + writer.sheets[DF['title']].cell(column=col+1,row=row,value=d) + writer.save() diff --git a/params.py b/params.py new file mode 100644 index 0000000..c3fafbc --- /dev/null +++ b/params.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- + +from sqlalchemy import create_engine + +isin_bdd = True +# Parametres bdd CEN38 OUT +user = 'postgres' +pwd = 'tutu' +adr = '192.168.60.10' +base = 'bd_cen' +con = create_engine('postgresql+psycopg2://{0}:{1}@{2}/{3}'.format(user,pwd,adr,base), echo=False) + +# Parametres bdd SICEN OUT +sicen_user = 'cen_admin' +sicen_pwd = '#CEN38@venir' +sicen_adr = '91.134.194.221' +sicen_base = 'sicen2' +sicen_con = create_engine('postgresql+psycopg2://{0}:{1}@{2}/{3}'.format(sicen_user,sicen_pwd,sicen_adr,sicen_base), echo=False) + + +DIC_REF_HAB = { + 'ssbv': { + 'gid':'id', + 'CD_SSBV': 'cdssbv', + 'LIB_SSBV': 'nom', + 'CD_COMGEO': 'cd_comgeo', + 'CD_CTB': 'cd_ctb', + 'CD_': 'cd_', + 'CD_SUBUNIT': 'cd_subunit', + 'geometry': 'geom', }, + 'plan_eau':{ + 'gid':'id', + 'geometry':'geom', + 'CdOH':'cdoh', + 'TopoOH':'nom', + 'NaturePE':'nature_pe',}, + 'masse_eau':{ + 'gid':'id', + 'geometry':'geom', + 'CdMasseDEa':'cd_mass_eau', + 'CdEuMasseD':'cdeu_mass_eau', + 'NomMasseDE':'nom', + 'SurfaceTot':'surfacetot', + 'CdEuSsBass':'cdeu_ssbass', + 'TypeGeneal':'type_geneal',}, + 'cours_eau':{ + 'gid':'id', + 'geometry':'geom', + 'CdOH':'cdoh', + 'TopoOH':'nom',}, + 'troncon_hydro':{ + 'gid':'id', + 'geometry':'geom', + 'CdOH':'cdoh', + 'TopoOH':'nom', + 'NatureTH':'nature_th', + 'TronconFic':'troncon_fic', + 'PositionPa':'position_pa', + 'Persistanc':'persistance', + 'OrigineTH':'origine_th', + 'SensEcoule':'sens_ecoule', + 'ReseauPrin':'reseau_prin', + 'Delimitati':'delimit', + 'ClasseLarg':'classe_larg', + 'BrasTH':'bras_th', + 'CdCoursEau':'cd_cours_eau', + 'CdSurfaceE':'cd_surf_eau',}, +} + +DIC_UNIQUE_KEY = { + 'ssbv': 'cdssbv', + 'plan_eau': 'cdoh', + 'masse_eau': 'cd_mass_eau', + 'cours_eau': 'cdoh', + 'troncon_hydro': 'nom', +} + + + +dic_zh_medwet = { + 'SITE_COD':'id', + 'DEPT':'dept', + 'ORG':'org', + 'NUM':'num', + 'SITE_NAME':'nom', +} + +class sites: + def tab_sites(): + tab = {} + tab['name'] = 'sites' + tab['schema'] = 'sites' + tab['desc_columns'] = con.dialect.get_columns(con,schema=tab['schema'],table_name=tab['name']) + tab['columns'] = [col['name'] for col in tab['desc_columns']] + tab['contraintes'] = con.dialect.get_foreign_keys(con,schema=tab['schema'],table_name=tab['name']) + tab['pkey'] = con.dialect.get_pk_constraint(con,schema=tab['schema'],table_name=tab['name']) + return tab + + def tab_geomsites(): + tab = {} + tab['name'] = 'r_sites_geom' + tab['schema'] = 'sites' + tab['desc_columns'] = con.dialect.get_columns(con,schema=tab['schema'],table_name=tab['name']) + tab['columns'] = [col['name'] for col in tab['desc_columns']] + tab['contraintes'] = con.dialect.get_foreign_keys(con,schema=tab['schema'],table_name=tab['name']) + tab['pkey'] = con.dialect.get_pk_constraint(con,schema=tab['schema'],table_name=tab['name']) + return tab + + + + + diff --git a/pers/__init__.py b/pers/__init__.py new file mode 100644 index 0000000..acda641 --- /dev/null +++ b/pers/__init__.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- +#Nom : : __init__.py +#Description : +#Copyright : 2021, CEN38 +#Auteur : Colas Geier +#Version : 1.0 + +from .pers import * \ No newline at end of file diff --git a/pers/__pycache__/__init__.cpython-38.pyc b/pers/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..2cfc26c Binary files /dev/null and b/pers/__pycache__/__init__.cpython-38.pyc differ diff --git a/pers/__pycache__/pers.cpython-38.pyc b/pers/__pycache__/pers.cpython-38.pyc new file mode 100644 index 0000000..b6d5eb1 Binary files /dev/null and b/pers/__pycache__/pers.cpython-38.pyc differ diff --git a/pers/pers.py b/pers/pers.py new file mode 100644 index 0000000..ede0e5e --- /dev/null +++ b/pers/pers.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- +#Nom : : pers.py +#Description : +#Copyright : 2021, CEN38 +#Auteur : Colas Geier +#Version : 1.0 + +from os import register_at_fork +from pandas import read_sql, merge, DataFrame +from pandas.core.indexing import is_nested_tuple +from ..tools import to_upper, to_tuple, to_upperfirst, _get_table,_aggr_cols +from ..params import con + +schema = 'personnes' + +def get_auteur(nom=None, prenom=None): + table = 'personne' + sql = 'SELECT * FROM %s.%s'%(schema,table) + if nom or prenom : sql = sql + ' WHERE ' + if nom : + sql = sql + 'nom IN %(nom)s' + nom = to_upper(nom) + if nom and prenom : sql = sql + ' AND ' + if prenom : + sql = sql + 'prenom IN %(prenom)s' + prenom = to_upperfirst(prenom) + df = read_sql( + sql = sql, + con = con, + params = {'nom': to_tuple(nom), 'prenom': to_tuple(prenom) }) + df['nom_prenom'] = df.nom + ' ' + df.prenom + df.loc[df.nom_prenom.isna(), 'nom_prenom'] = df.loc[df.nom_prenom.isna(), 'nom'] + + return df + +def get_organisme(ids=None, nom=None): + table = 'organisme' + df = _get_table(con, schema, table, ids=ids, nom=nom) + return df.set_index('id') + + +def _merge_orga(df, split_cols): + org = get_organisme() + aut = get_auteur() + df = df.copy() + for c in split_cols: + if not isinstance(df[c], int): df[c] = df[c].astype(float) + df[c].replace(aut.id.tolist(), aut.id_organisme.tolist(), inplace=True) + df[c].replace(org.id.tolist(), org.nom.tolist(), inplace=True) + df['organisme'] = None + for c in split_cols: + df.loc[df.organisme.isna(), 'organisme'] = df.loc[df['organisme'].isna(), c] + for c in split_cols: + comp = df.loc[~df[c].isna(),'organisme'].compare(df.loc[~df[c].isna(), c]) + if not comp.empty: + comp['test'] = comp.apply(lambda x: x['other'] in x['self'], axis=1) + comp = comp[~comp.test] + if not comp.empty: + df.loc[comp.index,'organisme'] = comp.self + ' & ' + comp.other + df.drop(columns=split_cols, inplace=True) + return df + + +def _merge_author(df, col_aut, orga=False, on_index=False): + # récupération des auteurs + aut = get_auteur().fillna('') + aut['nom_prenom'] = (aut['nom'] + ' ' + aut['prenom']).str.strip() + aut['id'] = aut['id'].astype(str) + id_inconnu = aut[aut.nom == 'INCONNU'].id[0] + + # merge des auteurs + df[col_aut].fillna(id_inconnu, inplace=True) + df[col_aut] = df[col_aut].replace(['\.0'],'',regex=True) + r_id = df[['id', col_aut]].copy() + r_idSplit = r_id[col_aut].str.split(' & ', expand=True) + r_id = r_id.join(r_idSplit) + cSplit = r_idSplit.shape[1] + cSplit = list(range(cSplit)) + + if orga: + # récup des organismes + org = _merge_orga(r_id, cSplit) + + r_id[cSplit] = r_id[cSplit].replace(aut['id'].tolist(),aut['nom_prenom'].tolist()) + r_id = _aggr_cols(r_id,cSplit,' & ') \ + .rename(columns={'aggreg': 'auteur'}) \ + .drop(columns=cSplit) + + if orga: + # merge des organismes + r_id = merge(r_id,org, on=['id', col_aut], suffixes=[None,'_y']) + + if on_index: + df = merge(df,r_id,how='left', right_index=True,left_index=True, suffixes=[None,'_y']) \ + .drop(columns=['id'+'_y',col_aut+'_y',col_aut]) + else: + df = merge(df,r_id,how='left', on=['id', col_aut], suffixes=[None,'_y']) \ + .drop(columns=[col_aut]) + + return df + + +def _merge_relation(df, table, schema, id=None, left_id=None,right_id=None): + if id: + params = {id: df[id].tolist() } + elif left_id and right_id: + params = {right_id: df[left_id].tolist() } + + mrg = _get_table(con, schema, table, params_col=params) + # if table == 'r_sites_auteur' or table == 'r_geomsites_auteur': + if table.startswith('r_') and table.endswith('_auteur'): + mrg = mrg[[right_id,'id_auteur']].groupby( + [right_id])['id_auteur'].apply(lambda x: ' & '.join(x.astype(str))) + mrg = DataFrame(data=mrg) + + if id: + df = merge(df,mrg, how='left', on=id) + elif left_id and right_id: + df = merge(df,mrg, how='left', left_on=left_id, right_on=right_id,suffixes=[None,'_y']) + rmcol = df.columns[df.columns.str.endswith('_y')].tolist() + if rmcol: + df.drop(columns=[*rmcol], inplace=True) + return df \ No newline at end of file diff --git a/ps.py b/ps.py new file mode 100644 index 0000000..6f5f442 --- /dev/null +++ b/ps.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- +#Nom : : ps.py +#Description : +#Copyright : 2021, CEN38 +#Auteur : Colas Geier +#Version : 1.0 + + + +from pandas.core.reshape.merge import merge +from .pers.pers import _get_table +from .sites.sites import _get_typ_milieux +# from .params import con +from .tools import _get_relation_tab,_set_geom + +schema = 'ps' +milieu = 'Pelouses sèches' + + +def get_param(): + from .tools import _get_param + return _get_param(schema=schema,param_table='param',type_table='type_param',type_court=False) + + + +def get_sitesInfos(id_site=None, nom_site=None, columns=None, with_nameOrga=False, statut='actif'): + from .sites.sites import get_sitesInfos + + df = get_sitesInfos( + ids=id_site, nom=nom_site, columns=columns, + with_nameOrga=with_nameOrga, milieu=milieu, statut=statut) + df.drop(columns=['typo_sdage'], inplace=True) + + return df + + +def get_listLegendePS(): + from .params import con + return _get_table(con,schema,'param_legende') + + +def get_sitesGeom(id_site=None, nom_site=None, columns=None, last_update=False, with_nameOrga=False, + params_col={}, statut='actif'): + from .sites.sites import get_sitesGeom + + df = get_sitesGeom( + id_site=id_site, nom_site=nom_site, columns=columns, + with_nameOrga=with_nameOrga,last_update=last_update, + params_col=params_col, milieu=milieu, statut=statut) + # df.drop(columns=['typo_sdage'], inplace=True) + + return df + + +def ps_r_site_param(id_site=None, nom_site=None, columns=None, last_update=False, + geom=False, raw=False, idparam=False, pivot=True): + from pandas import pivot_table + from .pers.pers import _merge_relation, _merge_author + + table = 'r_site_param' + r_tab_auteur = 'r_siteparam_auteur' + df = _get_relation_tab(schema=schema,tab=table,id_site=id_site,nom_site=nom_site, + last_update=last_update,geom=geom,milieu=milieu) + if 'auteur' in df.columns: + df.rename(columns={'auteur': 'auteur_geom'}, inplace=True) + df = _merge_relation(df=df,table=r_tab_auteur,schema=schema, left_id='id',right_id='id_siteparam') + df = _merge_author(df=df, col_aut='id_auteur') + df.rename(columns={'auteur': 'auteur_param'}, inplace=True) + + dic = get_param() + if not df.empty: + df.date_geom = df.date_geom.astype(str) + nval = 'param' + ntyp = 'type' + df = merge(df,dic, how='left', left_on='id_param', right_on='id', suffixes=(None,'_y'))\ + .drop(['id_y','id_param'],1) \ + .rename(columns={'description':'desc_param_ps', 'nom':nval}) + + if raw: + df.drop(columns='desc_param_ps', inplace=True) + else: + df.drop(columns=nval, inplace=True) + df.rename(columns={'desc_param_ps':nval}, inplace=True) + + df.taux.fillna(-999, inplace=True) + if geom: + dfgeom = df[['id_site', 'date_geom', 'geom']] + print(dfgeom) + dfgeom = dfgeom.drop_duplicates() + df.drop(columns=['geom'], inplace=True) + indexs = df.columns[~df.columns.isin(['taux',nval,'id','type'])].tolist() + + if pivot: + df2 = pivot_table( + df.fillna(''), + values=['taux',nval,'id'], + index=indexs, + columns=['type'], + # dropna=False, + aggfunc={lambda x: ', '.join(x.astype(str))}) + df2.columns = [ x[0].split('_')[0] + '_' + x[2][:6] for x in df2.columns ] + date = [x for x in df2.index.names if x.startswith('date_par')] + df2.reset_index(['auteur_param', *date], inplace=True) + df = df2 + else : + df.set_index(indexs, inplace=True) + + if geom: + idx = df.index.names + df.reset_index(inplace=True) + df = df.merge(dfgeom, how='left', on=['id_site', 'date_geom']) + df.set_index(idx, inplace=True) + df = _set_geom(df, hex=False) + + cols = df.columns[df.columns.str.startswith('param')].sort_values() + rcol = dic.type.unique() + rcol.sort() + dicc = dict(zip(cols,rcol)) + df.rename(columns=dicc, inplace=True) + + if not idparam: + rmcol = df.columns[df.columns.str.startswith('id')] + df.drop(columns=rmcol, inplace=True) + + df.replace(['-999',-999],[None, None],inplace=True, regex=True) + df.dropna(how='all', axis=1, inplace=True) + + return df + + + +def ps_r_site_habitat(id_site=None, nom_site=None, columns=None, last_update=False, + geom=False, raw=False, idhabitat=False, pivot=True): + from pandas import pivot_table + from .pers.pers import _merge_relation, _merge_author + + table = 'r_site_habitat' + r_tab_auteur = 'r_sitehab_auteur' + r_hab_cb = 'r_hab_cb' + + df = _get_relation_tab(schema=schema,tab=table,id_site=id_site, + nom_site=nom_site,last_update=last_update,geom=geom,milieu=milieu) + if 'auteur' in df.columns: + df.rename(columns={'auteur': 'auteur_geom'}, inplace=True) + + df = _merge_relation(df=df,table=r_tab_auteur,schema=schema, + left_id='id',right_id='id_sitehab') + df = _merge_author(df=df, col_aut='id_auteur') + df.rename(columns={'auteur': 'auteur_hab'}, inplace=True) + indexs = df.columns[~df.columns.isin(['id','n_hab','geom','index'])].tolist() + + df = _merge_relation(df=df,table=r_hab_cb,schema=schema, + left_id='id',right_id='id_sitehab') \ + .drop(columns=['id_sitehab']) + + if not raw: + from .zh import ref_hab + cb = ref_hab().get_CB(cols=['id','lb_hab_fr','descriptif_cb_fr']) + df = merge(df,cb, how='left', left_on='code_hab',right_on='id',suffixes=[None,'_y']) \ + .drop(columns=['code_hab','id_y']) \ + .rename(columns={'lb_hab_fr':'hab','descriptif_cb_fr':'desc_hab'}) + + if not df.empty: + df.date_geom = df.date_geom.astype(str) + if geom: + dfgeom = df[['id_site', 'date_geom', 'geom']] + print(dfgeom) + dfgeom = dfgeom.drop_duplicates() + df.drop(columns=['geom'], inplace=True) + + vals = df.columns[~df.columns.isin(['id',*indexs])].tolist() + if pivot: + df2 = pivot_table( + df.fillna(''), + values=vals, + index=indexs, + columns=['index'], + # dropna=False, + aggfunc={lambda x: ', '.join(x.astype(str)),}) + df2.columns = [ x[0] + str(x[2]) for x in df2.columns ] + date = [x for x in df2.index.names if x.startswith('date_hab')] + df2.reset_index(['auteur_hab', *date], inplace=True) + df = df2 + else : + df.set_index(indexs, inplace=True) + + if geom: + idx = df.index.names + df.reset_index(inplace=True) + df = df.merge(dfgeom, how='left', on=['id_site', 'date_geom']) + df.set_index(idx, inplace=True) + df = _set_geom(df, hex=False) + + if not idhabitat: + idx = df.columns[df.columns.str.startswith('id')] + df.drop(columns=idx, inplace=True) + + return df + + + +def get_ps(id_site=None, nom_site=None, columns=None, last_update=False, + geom=False, raw=[False,True], idparam=False, idhabitat=False): + + if isinstance(raw, bool): + rawp = rawh = raw + elif isinstance(raw, list): + rawp = raw[0] + rawh = raw[1] + + df = ps_r_site_param(id_site=id_site, nom_site=nom_site, columns=columns, + last_update=last_update, geom=False, raw=rawp, idparam=idparam, pivot=True) + df2 = ps_r_site_habitat(id_site=id_site, nom_site=nom_site, columns=columns, + last_update=last_update, geom=geom, raw=rawh, idhabitat=idhabitat, pivot=True) + + df = merge(df, df2, on=df.index.names) + + return df + diff --git a/ref/__init__.py b/ref/__init__.py new file mode 100644 index 0000000..c0a57e7 --- /dev/null +++ b/ref/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- + +from .territoire import * \ No newline at end of file diff --git a/ref/__pycache__/__init__.cpython-38.pyc b/ref/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..c01fc21 Binary files /dev/null and b/ref/__pycache__/__init__.cpython-38.pyc differ diff --git a/ref/territoire/__init__.py b/ref/territoire/__init__.py new file mode 100644 index 0000000..1c66a0b --- /dev/null +++ b/ref/territoire/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- + +from .territoire import get_districtNat \ No newline at end of file diff --git a/ref/territoire/__pycache__/__init__.cpython-38.pyc b/ref/territoire/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..b4eb026 Binary files /dev/null and b/ref/territoire/__pycache__/__init__.cpython-38.pyc differ diff --git a/ref/territoire/__pycache__/territoire.cpython-38.pyc b/ref/territoire/__pycache__/territoire.cpython-38.pyc new file mode 100644 index 0000000..bb0d0d1 Binary files /dev/null and b/ref/territoire/__pycache__/territoire.cpython-38.pyc differ diff --git a/ref/territoire/territoire.py b/ref/territoire/territoire.py new file mode 100644 index 0000000..2a72ec7 --- /dev/null +++ b/ref/territoire/territoire.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- + + +from ...params import con +from ...tools import _get_table +schema = 'ref_territoire' + + +def get_districtNat(): + return _get_table(con,schema,'districts_naturels') \ No newline at end of file diff --git a/sites/__init__.py b/sites/__init__.py new file mode 100644 index 0000000..e0a0508 --- /dev/null +++ b/sites/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- + diff --git a/sites/__pycache__/__init__.cpython-38.pyc b/sites/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..9517757 Binary files /dev/null and b/sites/__pycache__/__init__.cpython-38.pyc differ diff --git a/sites/__pycache__/sites.cpython-38.pyc b/sites/__pycache__/sites.cpython-38.pyc new file mode 100644 index 0000000..b21de50 Binary files /dev/null and b/sites/__pycache__/sites.cpython-38.pyc differ diff --git a/sites/sites.py b/sites/sites.py new file mode 100644 index 0000000..f2b2c8e --- /dev/null +++ b/sites/sites.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- +#Nom : : sites.py +#Description : +#Copyright : 2021, CEN38 +#Auteur : Colas Geier +#Version : 1.0 + +from ..tools import _get_table +from ..pers.pers import _get_table,_merge_author,_merge_relation + + + +from ..params import con +from ..pers import pers +schema = 'sites' +auteur = pers.get_auteur() +organisme = pers.get_organisme() +lst_tab = con.dialect.get_table_names(con,schema=schema) +columns_sitetab = con.dialect.get_columns(con,schema=schema,table_name='sites') +columns_geomtab = con.dialect.get_columns(con,schema=schema,table_name='r_sites_geom') + + +##################################### +### schema sites ### +##################################### +def _get_typ_milieux(ids=None, nom=None): + table = 'type_milieu' + df = _get_table(con, schema, table, ids=ids, nom=nom) + return df + +def _get_typo_sdage(ids=None, nom=None): + table = 'typo_sdage' + df = _get_table(con, schema, table, ids=ids, nom=nom) + return df + +def _get_typ_site(ids=None, nom=None): + table = 'type_site' + df = _get_table(con, schema, table, ids=ids, nom=nom) + return df + + +def get_sitesInfos(ids=None, nom=None, columns=None, with_nameOrga=False, details=False, + params_col={}, milieu=None, statut='actif'): + from pandas import merge + + drop = [] + table = 'sites' + typ_milieux = _get_typ_milieux() + typo_sdage = _get_typo_sdage() + typ_site = _get_typ_site() + + if milieu: + m = _get_typ_milieux(nom=milieu) + params_col = {**params_col, 'id_type_milieu':m.id[0].astype(str)} + + df = _get_table(con, schema, table, ids=ids, nom=nom, cols=columns, params_col=params_col, statut=statut) + + if not df.empty: + # récupération des auteurs + if 'id_auteur' in df.columns: + df.drop(columns='id_auteur', inplace=True) + df = _merge_relation(df=df,table='r_sites_auteur',schema=schema, left_id='id',right_id='id_site') + df = _merge_author(df=df, col_aut='id_auteur', orga=with_nameOrga) + # merge type_site + if 'id_type_site' in df.columns: + df = merge(df, typ_site, how='left', left_on='id_type_site', right_on='id', suffixes=('','_y') ) \ + .drop(columns=['id_type_site', 'id_y']) \ + .rename(columns={'nom_y': 'type_site', 'description': 'desc_type_site'}) + drop += ['desc_type_site'] + # merge typo_sdage + if 'id_typo_sdage' in df.columns: + df = merge(df, typo_sdage, how='left', left_on='id_typo_sdage', right_on='id', suffixes=('','_y') ) \ + .drop(columns=['id_typo_sdage', 'id_y']) \ + .rename(columns={'nom_y': 'typo_sdage', 'description': 'desc_typo_sdage'}) + drop += ['desc_typo_sdage'] + # merge type_milieu + if 'id_type_milieu' in df.columns: + df = merge(df, typ_milieux, how='left', left_on='id_type_milieu', right_on='id', suffixes=('','_y') ) \ + .drop(columns=['id_type_milieu', 'id_y']) \ + .rename(columns={'nom_y': 'type_milieu', 'description': 'desc_type_milieu', 'nom_court': 'nom_court_milieu'}) + drop += ['desc_type_milieu', 'nom_court_milieu'] + + if not details: + df.drop(columns=drop, inplace=True) + + return df.sort_values('id') + +def get_sitesGeom(id_site=None, nom_site=None, columns=None, last_update=False, + with_nameOrga=False, params_col={}, milieu=None, statut='actif'): + + if columns: + if not isinstance(columns, list): columns = [columns] + if 'id' not in columns: columns.insert(0,'id') + if 'id_site' not in columns: columns.insert(1,'id_site') + if 'geom' not in columns: columns.insert(2,'geom') + + if milieu: + m = _get_typ_milieux(nom=milieu) + params_col = {**params_col, 'id_type_milieu':m.id[0].astype(str)} + + table = 'sites' + df = _get_table(con, schema, table, ids=id_site, nom=nom_site, cols='id', params_col=params_col, statut=statut) + idSite = df.id.tolist() + if 'id_type_milieu' in params_col.keys(): + del params_col['id_type_milieu'] + + if idSite: + if 'id_site' in params_col.keys() : params_col['id_site'] = idSite + else: params_col = {**params_col, 'id_site':idSite} + table = 'r_sites_geom' + # df = _get_table(con, schema, table, params_col={'id_site':idSite}, cols=columns) + df = _get_table(con, schema, table, params_col=params_col, cols=columns) + if last_update: + df.sort_values(['id_site','date'], inplace=True) + df.reset_index(inplace=True, drop=True) + df.drop_duplicates(subset=['id_site'], keep='last', inplace=True) + df.reset_index(inplace=True, drop=True) + + if 'id_auteur' in df.columns: + df.drop(columns='id_auteur', inplace=True) + df = _merge_relation(df=df,table='r_geomsites_auteur',schema=schema, left_id='id',right_id='id_geom_site') + df = _merge_author(df=df, col_aut='id_auteur', orga=with_nameOrga) + + if 'auteur' in df.columns: + df.rename(columns={'auteur': 'auteur_geom'}, inplace=True) + + return df + +def new_site(df): + from ..tools.sites import tab_sites + final_col = [col['name'] for col in columns_sitetab] + aut = df['auteur'] + + + + + + +# lst_site = ['38RD0146','38RD0147','38RD0148','38RD0149','38RD0150','38RD0151','38RD0152', +# '38RD0153','38RD0155','38RD0156','38RD0157','38RD0158','38RD0159','38RD0160', +# '38RD0161','38RD0162','38RD0163','38RD0164','38RD0165'] +# df = zh.get_sitesInfos(id_site=lst_site) +# df.loc[df.auteur=='INCONNU','auteur'] = 'Gereco Environnement' \ No newline at end of file diff --git a/tools.py b/tools.py new file mode 100644 index 0000000..91a2af9 --- /dev/null +++ b/tools.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- +#Nom : : tools.py +#Description : +#Copyright : 2021, CEN38 +#Auteur : Colas Geier +#Version : 1.0 + +from pandas import Series, Index, read_sql, merge + + + +##################################### +### Fonctions générales ### +##################################### +def _aggr_cols(df, lst_col, sep=''): + df['aggreg'] = '' + for c,col in enumerate(lst_col): + add = '' + if c > 0: + add = sep + df.loc[~df[col].isna(),'aggreg'] = df.loc[~df[col].isna(),'aggreg'] + add + df.loc[~df[col].isna(),col] + return df + +def to_tuple(obj): + if isinstance(obj, (list,Series)): obj = tuple(obj) + if isinstance(obj, (int, str)) : obj = tuple([obj]) + return obj + +def to_colStringSQL(obj): + if isinstance(obj, (int)) : obj = str(obj) + if isinstance(obj, (list,Index)): obj = ",".join(obj) # Don't work with df.columns + return obj + +def to_upper(obj): + if isinstance(obj, Series): obj = Series([o.upper() for o in list(obj)]) + if isinstance(obj, tuple): obj = tuple([o.upper() for o in list(obj)]) + if isinstance(obj, list) : obj = [o.upper() for o in obj] + if isinstance(obj, str) : obj = obj.upper() + return obj + +def to_upperfirst(obj): + if isinstance(obj, Series): obj = Series([o.upper()[0] + o.lower()[1:] for o in list(obj)]) + if isinstance(obj, tuple): obj = tuple([o.upper()[0] + o.lower()[1:] for o in list(obj)]) + if isinstance(obj, list) : obj = [o.upper()[0] + o.lower()[1:] for o in obj] + if isinstance(obj, str) : obj = obj.upper()[0] + obj.lower()[1:] + return obj + +def dropZ(df,geom_col='geom'): + from shapely import wkb + df = df.copy() + df.loc[df[geom_col].has_z, geom_col] = [ + wkb.loads(wkb.dumps(geom, output_dimension=2)) for geom in df.loc[df[geom_col].has_z, geom_col] + ] + # if all(df_sites['geom'].has_z): + # # Suppression de la dimension Z + # geom_type = df_sites['geom'].geom_type + # df_sites['geom'] = [wkb.loads(wkb.dumps(geom, output_dimension=2)) for geom in df_sites['geom']] + # df_sites.set_geometry('geom', drop=False, inplace=True, crs=crs) + return df + +def remove_empty_keys(d): + for k in d.keys(): + if not d[k]: + del d[k] + +def _get_table(con, schema, table, ids=None, nom=None, cols=None, params_col={}, statut='actif'): + ''' + Selection d'une table: + + Parameters + ---------- + con : Connection sqlalchemy + schema : str. Nom du schema PostgreSQL. + table : str. Nom de la table PostgreSQL. + ids : liste. Identifiant de la table. + Doit contenir un champ 'id'. + nom : liste. Nom de la table. + Doit contenir un champ 'nom'. + cols : liste. Colonnes de la table à sélectionner. + params_col : Dict. Paramètre de Séléction IN. + requete : {'column': [liste]} + sql trad : 'column IN (liste)' + statut : str. 'actif', 'history', 'all'. + Statut des sites à récupérer, + 'actif'; Date_fin IS NULL + 'history'; A une Date_fin + 'all'; Tous les Sites + ''' + sql = 'SELECT * FROM {sch}.{tab}'.format(sch=schema, tab=table) + if params_col: + params_col = { k: v for k, v in params_col.items() if v } + + if cols : sql = sql.replace('*', to_colStringSQL(cols) ) + # Si arg (ids|nom|params_col), ajout de 'WHERE' + if ids or nom or params_col or (statut!='all' and table=='sites') : sql = sql + ' WHERE ' + if ids : sql = sql + 'id IN %(ids)s' + if ids and (nom or params_col or (statut!='all' and table=='sites')) : sql = sql + ' AND ' + if nom : sql = sql + 'nom IN %(nom)s' + if nom and (params_col or (statut!='all' and table=='sites')) : sql = sql + ' AND ' + if statut=='actif' and table=='sites': sql = sql + ' date_fin IS NULL ' + if statut=='history' and table=='sites': sql = sql + ' date_fin IS NOT NULL ' + if params_col and (statut!='all' and table=='sites') : sql = sql + ' AND ' + if params_col : + sql = sql + ' AND '.join([k + ' IN %({})s'.format(k) for k in params_col.keys()]) + params_col = {key:to_tuple(params_col[key]) for key in params_col.keys()} + + sql = sql.replace("',)","')") + df = read_sql( + sql = sql, + con = con, + params = {'ids': to_tuple(ids), 'nom': to_tuple(nom), **params_col }) + if 'geom' in df.columns: + df = _set_geom(df) + return df + +def _set_geom(df, hex=True): + from shapely.wkb import loads + import geopandas as gpd # set_geometry + + if hex: + df['geom'] = [(loads(geom, hex=True)) for geom in df['geom']] + df = df.set_geometry('geom', crs='EPSG:2154') + + return df + + +def _get_param(schema, param_table, type_table=None, type_court=True): + from .params import con + + if type_table: + typ = _get_table(con, schema, table=type_table) + par = _get_table(con, schema, table=param_table, params_col={'id_type':typ.id.tolist()}) + df = merge(par, typ, left_on='id_type', right_on='id', how='left', suffixes=(None, '_typ')) \ + .drop(columns=['id_type','id_typ']) + if 'description_typ' in df.columns: del df['description_typ'] + if type_court: df = df.drop(columns=['nom_typ']).rename(columns={'nom_court_typ':'type'}) + else : df = df.drop(columns=['nom_court_typ'],errors='ignore').rename(columns={'nom_typ':'type'}) + index = ['id'] + if 'type' in df.columns: + index += ['type'] + df = df.set_index(index).reset_index() + else: + df = _get_table(con, schema, table=param_table) + + return df + + +def _get_relation_tab(schema, tab, id_site=None, nom_site=None, last_update=False, + geom=False,params_col={},milieu=None,statut='actif'): + ''' + + Parameters + ---------- + schema : str. Default : None. + Schéma de la database New_cen38. + tab : str. Default : None. + Schéma de la database New_cen38. + id_site : str,list. Default : None. + Identifiants des sites présent dans la table 'sites'. + nom_site : str,list. + Nom des sites présent dans la table 'sites'. + last_update : bool. Default : False. + If True, récupération des dernières données à jour. + If False, récupération des toutes les données. + geom : bool. Default : False. + Return les geometries des sites + params_col : dict. Default : {}. + Application des conditions de séléction des données + sous la forme d'un dictionnaire {'nomcolumn': conditions}. + milieu : str. Default : None. + Nom d'un milieu référencé dans la table `sites.type_milieu`. + Liste récupérable avec la fonction `pyzh.sites._get_typ_milieux()` + statut : str. 'actif', 'history', 'all'. + Statut des sites à récupérer, + 'actif'; Date_fin IS NULL + 'history'; A une Date_fin + 'all'; Tous les Sites + + + Return + ---------- + df + ''' + from .params import con + from .sites.sites import get_sitesGeom + # from .pers.pers import _merge_relation, _merge_author + + table = 'sites' + dfSG = get_sitesGeom(columns='date', id_site=id_site, nom_site=nom_site, + last_update=last_update,params_col=params_col,milieu=milieu,statut=statut) + + if not geom and not dfSG.empty: + dfSG.drop('geom',1,inplace=True) + ids = dfSG.id.tolist() + table = tab + + if ids : + df = _get_table(con, schema, table, params_col={'id_geom_site':ids}) + if last_update: + tmp = ['id', 'date', 'valid'] + col = [*df.columns[~df.columns.isin(tmp)]] + df = df.sort_values(col).reset_index(drop=True) + df.drop_duplicates(subset=col, keep='last', inplace=True) + df.reset_index(drop=True, inplace=True) + df = df[df.valid].copy() + + if 'date' in dfSG.columns and 'date' in df.columns: + dfSG.rename(columns={'date':'date_geom'}, inplace=True) + df.rename(columns={'date':'date_'+table.rsplit('_',1)[1][:5]}, inplace=True) + # if table == 'r_site_sub': + # print('DF : {}'.format(df)) + # print(df.empty) + # print('DFSG : {}'.format(dfSG)) + + if not df.empty: + df = merge(dfSG,df, how='left', left_on='id', right_on='id_geom_site', suffixes=('_x', None)) \ + .drop(['id_x','id_geom_site'],1) \ + .set_index('id').reset_index() + + return df + else: + print('PAS de géometries de sites sélectionnées ...') + + +def _get_relation_autor(df, relation_tab, schema, id_df, id_relation, id_rela_auth='id_auteur'): + from .pers.pers import _merge_relation, _merge_author + if 'site' in relation_tab: + suffixe = relation_tab.split('_')[1].split('site')[1] + suffixe = '_' + suffixe + df = _merge_relation(df=df,table=relation_tab,schema=schema, + left_id = id_df, + right_id = id_relation) + df = _merge_author(df=df, col_aut=id_rela_auth, on_index=True) + df.rename(columns={'auteur': 'auteur'+suffixe}, inplace=True) + return df + + + +def to_geoms(geometries): + from shapely.geometry import Polygon,LineString + for geometry in geometries: + if isinstance(geometry, (Polygon,LineString)): + yield geometry + else: + yield from geometry + + +def union_polygons_geometry(df): + ''' + Transforme un GeoDataFrame de Polygons + et/ou MultiPolygons en un MultiPolygon unique: + + Parameters + ---------- + df : GeoDataFrame. + ''' + from shapely.geometry import MultiPolygon + name_geom = df.geometry.name + + poly = df.loc[df.geom_type=='Polygon',name_geom].tolist() + multipoly = df.loc[df.geom_type=='MultiPolygon',name_geom].tolist() + + if poly: + mp2 = MultiPolygon(poly) + if poly and multipoly: + res = MultiPolygon(to_geoms([*mp2, *multipoly])) + elif not poly and multipoly: + res = MultiPolygon(to_geoms(multipoly)) + elif not multipoly and poly: + res = MultiPolygon(poly) + + return res + + +def union_lines_geometry(df): + from shapely.geometry import MultiLineString + name_geom = df.geometry.name + + line = df.loc[df.geom_type=='LineString',name_geom].tolist() + multiline = df.loc[df.geom_type=='MultiLineString',name_geom].tolist() + + if line: + mp2 = MultiLineString(line) + if line and multiline: + res = MultiLineString(to_geoms([*mp2, *multiline])) + elif not line and multiline: + res = MultiLineString(to_geoms([*multiline])) + elif not multiline and line: + res = MultiLineString(line) + + return res + + +def calc_recouvrmt(df1,df2): + ''' + Calcule le recouvrement de df2 sur df1 + pour chaque géométrie de df1: + + Parameters + ---------- + df1 : GeoDataFrame. + df2 : GeoDataFrame. + ''' + from geopandas import sjoin + tmp = sjoin( + df1, + df2[['geom']], + op = 'intersects', + how = 'left') + tmp.dropna(subset=['index_right'],inplace=True) + tmp.index_right = tmp.index_right.astype(int) + tmp.reset_index(inplace=True) + tmp = tmp.join( + df2[['geom']].rename(columns={'geom': 'right_geom'}), + on=['index_right'], how='left') + tmp2 = tmp[['index_right','right_geom']].copy() \ + .rename(columns={'right_geom': 'geom'}) \ + .set_geometry('geom') + tmp1 = tmp[['id_site','geom']].copy() \ + .set_geometry('geom') + + if not tmp1.geom.values.is_valid.all(): + tmp1.loc[~tmp1.geom.values.is_valid,'geom'] = tmp1.loc[~tmp1.geom.values.is_valid,'geom'].buffer(0) + if not tmp2.geom.values.is_valid.all(): + tmp2.loc[~tmp2.geom.values.is_valid,'geom'] = tmp2.loc[~tmp2.geom.values.is_valid,'geom'].buffer(0) + + tmp['perc_rcvmt'] = (tmp1.intersection(tmp2).area/tmp1.area)*100 + tmp = tmp.groupby(['id_site']).sum().reset_index() + df1 = df1.merge(tmp[['id_site','perc_rcvmt']], on=['id_site'], how='left') + df1.perc_rcvmt.fillna(0, inplace=True) + df1.perc_rcvmt = df1.perc_rcvmt.round(2) + return df1 + + +def Polygons_to_MultiPolygon(df): + from shapely.geometry import MultiPolygon + from pandas import concat + df = df.copy() + multi = df.loc[df.geom_type=='MultiPolygon'].copy() + poly = df.loc[df.geom_type=='Polygon'].copy() + poly['geom'] = [MultiPolygon([geom]) for geom in df.loc[df.geom_type=='Polygon','geom'] ] + df = concat([multi,poly]) + df.sort_index(inplace=True) + return df \ No newline at end of file diff --git a/update.py b/update.py new file mode 100644 index 0000000..3b4c754 --- /dev/null +++ b/update.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- +#Nom : : update.py +#Description : +#Copyright : 2021, CEN38 +#Auteur : Colas Geier +#Version : 1.0 +""" +Module contains tools for processing files into DataFrames or other objects +""" + +import geopandas as gpd +import pandas as pd +from .params import DIC_REF_HAB, DIC_UNIQUE_KEY +from .tools import _get_table + +##################################### +### Update ### +##################################### +def __get_pkey__(engine,table_name,schema): + pk = engine.dialect.get_pk_constraint(engine,table_name=table_name,schema=schema) + return pk + +def _get_dic(self): + if self._table: + select_cols = DIC_REF_HAB[self._table] + return select_cols + +def _get_schema_name(self): + engine = self.con + lst_sch = engine.dialect.get_schema_names(engine) + tab = [] + for sch in lst_sch: + lst_tab = engine.dialect.get_table_names(engine,schema=sch) + if self._table in lst_tab: + tab += [sch] + + if len(tab) > 1: + print('La table %s exitent dans plusieurs schéma ! Préciser l''argument ''schema'''%self._table) + elif len(tab) == 0: + print('La table %s sitée n''existe pas ...') + else: + return tab[0] + +def _check_data_exist(self, df): + engine = self.con + pk = __get_pkey__(engine,table_name=self._table,schema=self._schema) + self._pkey = pk['constrained_columns'][0] + pkey = self._pkey + if self._table in DIC_UNIQUE_KEY.keys(): + pkey = DIC_UNIQUE_KEY[self._table] + + data_indb = _get_table(self.con, self._schema, self._table, cols=[pkey]) + data_exist_not = df[~df[pkey].isin(data_indb[pkey])] + data_exist = df[df[pkey].isin(data_indb[pkey])] + data_exist_eq, data_exist_ne = _check_eq_df(self, data_exist) + + return [data_exist_eq, data_exist_ne, data_exist_not] + + +def _check_eq_df(self,df): + from .zh import _get_table + pkey = self._pkey + params = {} + if isinstance(pkey, list): + for key in pkey: + p = {key: df[key]} + params = {**params, **p} + else: + params = {pkey:df[pkey]} + data = _get_table(self.con, self._schema, self._table, cols=df.columns.tolist(), params_col=params) + if self._table in DIC_UNIQUE_KEY.keys(): + if DIC_UNIQUE_KEY[self._table] != pkey: + df = df.sort_values(DIC_UNIQUE_KEY[self._table]) + data = data.sort_values(DIC_UNIQUE_KEY[self._table]) + else : + df = df.sort_values(pkey) + data = data.sort_values(pkey) + else: + df = df.sort_values(pkey) + data = data.sort_values(pkey) + if self._table in DIC_UNIQUE_KEY.keys(): + if DIC_UNIQUE_KEY[self._table] != pkey: + df.set_index(pkey, inplace=True) + data.set_index(pkey, inplace=True) + eq = df[df.eq(data).all(axis=1)] + ne = df[df.ne(data).any(axis=1)] + eq.reset_index(drop=False, inplace=True) + ne.reset_index(drop=False, inplace=True) + else: + eq = df[df.eq(data).all(axis=1)] + ne = df[df.ne(data).any(axis=1)] + else: + eq = df[df.eq(data).all(axis=1)] + ne = df[df.ne(data).any(axis=1)] + return [eq,ne] + + def __openFile__(file): + if isinstance(file,str): + df = pd.read_table(file) + + + +class update_ref_table: + ''' + Mise à jour des tables de référance situées dans les + schéma : + ref_habitats, ref_hydro + + Paramètres : + ------------ + file : str. chemain du vecteur. + table : str. Nom de la table à mettre à jour. + schema : str. Nom du schéma. + update : bool. Si True, mise à jour des champs différents. + Défault False. + ''' + + def __init__(self, file, table, schema=None, update=False): + + from .params import con + + self.con = con + self._file = file + self._table = table + self._updt = update + self._schema = schema + + if not self._schema: + self._schema = _get_schema_name(self) + self._pkey = __get_pkey__( + self.con,table_name=self._table,schema=self._schema + )['constrained_columns'][0] + + self._update_table_with_geom() + + + def _update_table_with_geom(self): + dic = _get_dic(self) + select_cols = list(dic.values()) + + if isinstance(self._file,str): + df = gpd.read_file(self._file) + elif isinstance(self._file,gpd.GeoDataFrame): + df = self._file.copy() + else: + print('Argument ''file'' ERROR !') + + if 'id' not in df.columns: + df.index.name = 'id' + df.reset_index(drop=False, inplace=True) + + if not df.crs: + df.set_crs(epsg=2154, inplace=True) + + if df.crs.srs != 'epsg:2154': + df.to_crs(epsg=2154, inplace=True) + + df.rename(columns=dic, inplace=True) + df = df[select_cols] + + df = df.set_geometry('geom') + + if 'Polygon' in df.geom_type.unique() and 'MultiPolygon' in df.geom_type.unique(): + from shapely.geometry.multipolygon import MultiPolygon + tmp = df.loc[df.geom_type == 'Polygon'].copy() + geom = [MultiPolygon([x]) for x in tmp.loc[tmp.geom_type == 'Polygon','geom']] + tmp = tmp.set_geometry(geom) + df = pd.concat([df.drop(tmp.index), tmp]).sort_values('id') + + if 'LineString' in df.geom_type.unique() and 'MultiLineString' in df.geom_type.unique(): + from shapely.geometry.multilinestring import MultiLineString + tmp = df.loc[df.geom_type == 'LineString'].copy() + geom = [MultiLineString([x]) for x in tmp.loc[tmp.geom_type == 'LineString','geom']] + tmp = tmp.set_geometry(geom) + df = pd.concat([df.drop(tmp.index), tmp]).sort_values('id') + + df_exst, df_updt, df_imp = _check_data_exist(self,df) + len_imp = len(df_imp) + len_updt = len(df_updt) + len_exst = len(df_exst) + if len_exst > 0: + print('DATAS {0}/{1} already exist'.format(len_exst,len(df))) + else: + print('NO DATA EXIST ...') + if df_imp.empty: + print('NO DATA TO IMPORT ...') + else : + Q = input('IMPORT {0}/{1} ... Yes(y)/No(n)/View(v) ?'.format(len_imp,len(df))) + while Q.lower() in ['view','v']: + print(df_imp) + Q = input('IMPORT {0}/{1} ... Yes(y)/No(n)/View(v) ?'.format(len_imp,len(df))) + if Q.lower() in ['yes','y']: + print('IMPORT {0}/{1} ...'.format(len_imp,len(df))) + df_imp.to_postgis( + name = self._table, + con = self.con, + schema = self._schema, + if_exists = 'append', + geom_col = 'geom' + ) + elif Q.lower() in ['no','n']: + pass + + if df_updt.empty: + print('NO DATA TO UPDATE ...') + else : + Q = input('UPDATE {0}/{1} ... Yes(y)/No(n)/View(v) ?'.format(len_updt,len(df))) + while Q.lower() in ['view','v']: + print(df_updt) + Q = input('UPDATE {0}/{1} ... Yes(y)/No(n)/View(v) ?'.format(len_updt,len(df))) + if Q.lower() in ['yes','y']: + print('UPDATE {0}/{1} ...'.format(len_updt,len(df))) + update.update_to_sql( + df=df_updt, + con=self.con, + table_name=self._table, + schema_name=self._schema, + key_name=self._pkey + ) + elif Q.lower() in ['no','n']: + pass + + + +class update: + def update_ref(file, table, schema=None, update=False): + update_ref_table(file, table, schema, update) + + + def update_to_sql(df, con, table_name, schema_name, key_name): + from sys import exit + + a = [] + b = [] + table = table_name + schema = schema_name + primary_key = key_name + pkey = __get_pkey__( + con,table_name=table,schema=schema) + + if pkey not in df.columns: + exit('Le champs clé primaire "%s" ne figure pas dans le DataFrame'%pkey) + + if isinstance(primary_key, str): + primary_key = [primary_key] + + for col in df.columns: + if col in primary_key: + b.append("t.{col}=f.{col}".format(col=col)) + else: + a.append("{col}=t.{col}".format(col=col)) + + if isinstance(df, gpd.GeoDataFrame): + df.to_postgis( + name = 'temp_table', + con = con, + schema = schema, + if_exists = 'replace', + geom_col = df.geometry.name + ) + else: + df.to_sql( + name = 'temp_table', + con = con, + schema = schema, + if_exists = 'replace', + index = False, + method = 'multi' + ) + + update_stmt_1 = "UPDATE {sch}.{final_table} f".format(sch=schema,final_table=table) + update_stmt_2 = " FROM {sch}.temp_table t".format(sch=schema) + update_stmt_6 = " WHERE %s"%' AND '.join(b) + update_stmt_3 = " SET " + update_stmt_4 = ", ".join(a) + update_stmt_5 = update_stmt_1 + update_stmt_3 + update_stmt_4 + update_stmt_2 + update_stmt_6 + ";" + drop_stmt = "DROP TABLE {sch}.temp_table ;".format(sch=schema) + with con.begin() as cnx: + cnx.execute(update_stmt_5) + cnx.execute(drop_stmt) + return print('END update') \ No newline at end of file diff --git a/vrac/Modif_caract_connect.py b/vrac/Modif_caract_connect.py new file mode 100644 index 0000000..d240f17 --- /dev/null +++ b/vrac/Modif_caract_connect.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- +import pandas as pd +import pyzh +zh = pyzh.zh() + +con = pyzh.con +sch = 'zones_humides' +lst_site_old = ['38RD0146','38RD0147'] +lst_site_new = ['38RD0164','38RD0165'] +df = zh.get_connex(id_site=lst_site_old) +geomsite = zh.get_sitesGeom(id_site=lst_site_new) +aut = pyzh.pers.get_auteur() + +tab = 'r_site_type_connect' +rtab_aut = 'r_rsiteconnect_auteur' +id_rsiteaut = 'id_siteconnect' +date_champ = 'date_conne' +nom_champ = 'connexion' +id_champ = 'id_param_connect' +delim = zh._get_param(param_table='param_type_connect', type_court=False) + + +df.drop( + columns=['id','valid', 'date_geom', 'auteur_geom', 'description','type','lb_hab_fr'], + inplace=True, + errors='ignore') + +for i, s in enumerate(lst_site_new): + df.loc[df.id_site==lst_site_old[i], 'id_site'] = s + df.loc[df.id_site==s, ['id_geom_site']] = geomsite.loc[geomsite.id_site==s,'id'].values[0].astype(str) + +df['id_geom_site'] = df.id_geom_site.astype(int) +df[id_champ] = df[nom_champ] +df[id_champ] = df[id_champ].replace(delim.nom.tolist(),delim.id.tolist()) +df['date'] = df[date_champ] + +df.drop(columns=['id_site',nom_champ,date_champ], inplace=True) +lst_idgeom = df.id_geom_site.unique().astype(str) + +# check si data in bdd +sql = 'select exists(select * from {sch}.{tab} where id_geom_site in ({lst_id}));'.format( + sch=sch, tab=tab, lst_id=','.join(lst_idgeom) +) +res = pd.read_sql(sql,con) +if res.exists[0]: + # delete if exist + sql = 'delete from {sch}.{tab} where id_geom_site in ({lst_id});'.format( + sch=sch, tab=tab, lst_id=','.join(lst_idgeom) + ) + pd.read_sql(sql,con) +# insert new line in bdd +col = df.columns[~df.columns.str.contains('auteur')] +colaut = df.columns[df.columns.str.contains('auteur')] +df[col].to_sql( + tab, con, schema=sch, + if_exists='append',index=False,method='multi' +) +# Récupération des nouveaux ids insérés +sql = 'select * from {sch}.{tab} where id_geom_site in ({lst_id});'.format( + sch=sch, tab=tab, lst_id=','.join(lst_idgeom) +) +res = pd.read_sql(sql,con) +df = df.merge(res, how='left', on=[*col]) + +ins_aut = df[['id',*colaut]].copy() +aut['nom_prenom'] = aut.nom + ' ' + aut.prenom +aut.loc[aut.nom_prenom.isna(), 'nom_prenom'] = aut.loc[aut.nom_prenom.isna(), 'nom'] +ins_aut[colaut] = ins_aut[colaut].replace(aut.nom_prenom.tolist(), aut.id.tolist()) +ins_aut.columns = [id_rsiteaut, 'id_auteur'] +ins_aut.to_sql( + rtab_aut, con, schema=sch, + if_exists='append',index=False,method='multi') \ No newline at end of file diff --git a/vrac/Modif_caract_delim.py b/vrac/Modif_caract_delim.py new file mode 100644 index 0000000..3be32de --- /dev/null +++ b/vrac/Modif_caract_delim.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- +import pandas as pd +import pyzh +zh = pyzh.zh() + +con = pyzh.con +tab = 'r_site_critdelim' +sch = 'zones_humides' + +lst_site_old = ['38RD0146','38RD0147'] +lst_site_new = ['38RD0164','38RD0165'] + +df = zh.get_delim(id_site=lst_site_old) +geomsite = zh.get_sitesGeom(id_site=lst_site_new) +delim = zh._get_param(param_table='param_delim_fct',type_table='type_param_delim_fct') +aut = pyzh.pers.get_auteur() + +df.drop(columns=['id','valid', 'date_geom', 'auteur_geom', 'desc_param','type'], inplace=True) +for i, s in enumerate(lst_site_new): + df.loc[df.id_site==lst_site_old[i], 'id_site'] = s + df.loc[df.id_site==s, ['id_geom_site']] = geomsite.loc[geomsite.id_site==s,'id'].values[0].astype(str) + +df['id_geom_site'] = df.id_geom_site.astype(int) +df['id_crit_delim'] = df.nom_crit +df['id_crit_delim'] = df['id_crit_delim'].replace(delim.nom.tolist(),delim.id.tolist()) +df['date'] = df.date_critd + + +df.drop(columns=['id_site','nom_crit','date_critd'], inplace=True) +lst_idgeom = df.id_geom_site.unique().astype(str) + +sql = 'select exists(select * from {sch}.{tab} where id_geom_site in ({lst_id}));'.format( + sch=sch, tab=tab, lst_id=','.join(lst_idgeom) +) +res = pd.read_sql(sql,con) +if res.exists[0]: + sql = 'delete from {sch}.{tab} where id_geom_site in ({lst_id});'.format( + sch=sch, tab=tab, lst_id=','.join(lst_idgeom) + ) + pd.read_sql(sql,con) +col = df.columns[~df.columns.str.contains('auteur')] +colaut = df.columns[df.columns.str.contains('auteur')] +df[col].to_sql( + tab, con, schema=sch, + if_exists='append',index=False,method='multi' +) +sql = 'select * from {sch}.{tab} where id_geom_site in ({lst_id});'.format( + sch=sch, tab=tab, lst_id=','.join(lst_idgeom) +) +res = pd.read_sql(sql,con) +df = df.merge(res, how='left', on=[*col]) + +ins_aut = df[['id',*colaut]].copy() +aut['nom_prenom'] = aut.nom + ' ' + aut.prenom +aut.loc[aut.nom_prenom.isna(), 'nom_prenom'] = aut.loc[aut.nom_prenom.isna(), 'nom'] +ins_aut[colaut] = ins_aut[colaut].replace(aut.nom_prenom.tolist(), aut.id.tolist()) +ins_aut.columns = ['id_sitedelim', 'id_auteur'] +ins_aut.to_sql( + 'r_rsitedelim_auteur', con, schema=sch, + if_exists='append',index=False,method='multi') + + diff --git a/vrac/Modif_caract_fct.py b/vrac/Modif_caract_fct.py new file mode 100644 index 0000000..4da6f1d --- /dev/null +++ b/vrac/Modif_caract_fct.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- +import pandas as pd +import pyzh +zh = pyzh.zh() + +con = pyzh.con +sch = 'zones_humides' +lst_site_old = ['38RD0146','38RD0147'] +lst_site_new = ['38RD0164','38RD0165'] +df = zh.get_fct(id_site=lst_site_old) +geomsite = zh.get_sitesGeom(id_site=lst_site_new) +aut = pyzh.pers.get_auteur() + +tab = 'r_site_fctecosociopatri' +rtab_aut = 'r_rsitefct_auteur' +id_rsiteaut = 'id_sitefct' +delim = zh._get_param(param_table='param_fct_eco_socio_patri',type_table='type_param_fct') + + +df.drop( + columns=['id','valid', 'date_geom', 'auteur_geom', 'desc_param','type'], + inplace=True, + errors='ignore') + +for i, s in enumerate(lst_site_new): + df.loc[df.id_site==lst_site_old[i], 'id_site'] = s + df.loc[df.id_site==s, ['id_geom_site']] = geomsite.loc[geomsite.id_site==s,'id'].values[0].astype(str) + +df['id_geom_site'] = df.id_geom_site.astype(int) +df['id_fct'] = df['nom_fct'] +df['id_fct'] = df['id_fct'].replace(delim.nom.tolist(),delim.id.tolist()) +df['date'] = df.date_fctec + +df.drop(columns=['id_site','nom_fct','date_fctec'], inplace=True) +lst_idgeom = df.id_geom_site.unique().astype(str) + +# check si data in bdd +sql = 'select exists(select * from {sch}.{tab} where id_geom_site in ({lst_id}));'.format( + sch=sch, tab=tab, lst_id=','.join(lst_idgeom) +) +res = pd.read_sql(sql,con) +if res.exists[0]: + # delete if exist + sql = 'delete from {sch}.{tab} where id_geom_site in ({lst_id});'.format( + sch=sch, tab=tab, lst_id=','.join(lst_idgeom) + ) + pd.read_sql(sql,con) +# insert new line in bdd +col = df.columns[~df.columns.str.contains('auteur')] +colaut = df.columns[df.columns.str.contains('auteur')] +df[col].to_sql( + tab, con, schema=sch, + if_exists='append',index=False,method='multi' +) +# Récupération des nouveaux ids insérés +sql = 'select * from {sch}.{tab} where id_geom_site in ({lst_id});'.format( + sch=sch, tab=tab, lst_id=','.join(lst_idgeom) +) +res = pd.read_sql(sql,con) +df = df.merge(res, how='left', on=[*col]) + +ins_aut = df[['id',*colaut]].copy() +aut['nom_prenom'] = aut.nom + ' ' + aut.prenom +aut.loc[aut.nom_prenom.isna(), 'nom_prenom'] = aut.loc[aut.nom_prenom.isna(), 'nom'] +ins_aut[colaut] = ins_aut[colaut].replace(aut.nom_prenom.tolist(), aut.id.tolist()) +ins_aut.columns = [id_rsiteaut, 'id_auteur'] +ins_aut.to_sql( + rtab_aut, con, schema=sch, + if_exists='append',index=False,method='multi') \ No newline at end of file diff --git a/vrac/Modif_caract_hab.py b/vrac/Modif_caract_hab.py new file mode 100644 index 0000000..0709427 --- /dev/null +++ b/vrac/Modif_caract_hab.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- +import pandas as pd +import pyzh +zh = pyzh.zh() + +con = pyzh.con +sch = 'zones_humides' +lst_site_old = ['38RD0146','38RD0147'] +lst_site_new = ['38RD0164','38RD0165'] +df = zh.get_habitat(id_site=lst_site_old) +geomsite = zh.get_sitesGeom(id_site=lst_site_new) +aut = pyzh.pers.get_auteur() + +tab = 'r_site_habitat' +rtab_aut = 'r_rsitehab_auteur' +id_rsiteaut = 'id_sitehab' +date_champ = 'date_habit' +nom_champ = 'code_cb' +id_champ = 'id_cb' +# delim = zh._get_param(param_table='param_fct_eco_socio_patri',type_table='type_param_fct') + + +df.drop( + columns=['id','valid', 'date_geom', 'auteur_geom', 'desc_param','type','lb_hab_fr'], + inplace=True, + errors='ignore') + +for i, s in enumerate(lst_site_new): + df.loc[df.id_site==lst_site_old[i], 'id_site'] = s + df.loc[df.id_site==s, ['id_geom_site']] = geomsite.loc[geomsite.id_site==s,'id'].values[0].astype(str) + +df['id_geom_site'] = df.id_geom_site.astype(int) +df[id_champ] = df[nom_champ] +# df[id_champ] = df[id_champ].replace(delim.nom.tolist(),delim.id.tolist()) +df['date'] = df[date_champ] + +df.drop(columns=['id_site',nom_champ,date_champ], inplace=True) +lst_idgeom = df.id_geom_site.unique().astype(str) + +# check si data in bdd +sql = 'select exists(select * from {sch}.{tab} where id_geom_site in ({lst_id}));'.format( + sch=sch, tab=tab, lst_id=','.join(lst_idgeom) +) +res = pd.read_sql(sql,con) +if res.exists[0]: + # delete if exist + sql = 'delete from {sch}.{tab} where id_geom_site in ({lst_id});'.format( + sch=sch, tab=tab, lst_id=','.join(lst_idgeom) + ) + pd.read_sql(sql,con) +# insert new line in bdd +col = df.columns[~df.columns.str.contains('auteur')] +colaut = df.columns[df.columns.str.contains('auteur')] +df[col].to_sql( + tab, con, schema=sch, + if_exists='append',index=False,method='multi' +) +# Récupération des nouveaux ids insérés +sql = 'select * from {sch}.{tab} where id_geom_site in ({lst_id});'.format( + sch=sch, tab=tab, lst_id=','.join(lst_idgeom) +) +res = pd.read_sql(sql,con) +df = df.merge(res, how='left', on=[*col]) + +ins_aut = df[['id',*colaut]].copy() +aut['nom_prenom'] = aut.nom + ' ' + aut.prenom +aut.loc[aut.nom_prenom.isna(), 'nom_prenom'] = aut.loc[aut.nom_prenom.isna(), 'nom'] +ins_aut[colaut] = ins_aut[colaut].replace(aut.nom_prenom.tolist(), aut.id.tolist()) +ins_aut.columns = [id_rsiteaut, 'id_auteur'] +ins_aut.to_sql( + rtab_aut, con, schema=sch, + if_exists='append',index=False,method='multi') \ No newline at end of file diff --git a/vrac/Modif_caract_reghydro.py b/vrac/Modif_caract_reghydro.py new file mode 100644 index 0000000..830acbf --- /dev/null +++ b/vrac/Modif_caract_reghydro.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- +import pandas as pd +import pyzh +zh = pyzh.zh() + +con = pyzh.con +sch = 'zones_humides' +lst_site_old = ['38RD0146','38RD0147'] +lst_site_new = ['38RD0164','38RD0165'] +df = zh.get_regHydro(id_site=lst_site_old) +geomsite = zh.get_sitesGeom(id_site=lst_site_new) +aut = pyzh.pers.get_auteur() + + +tab = 'r_site_reghydro' +rtab_aut = 'r_rsitehydro_auteur' +id_rsiteaut = 'id_sitehydro' +date_champ = 'date_reghy' +nom_champ1 = 'regime_hydri' +nom_champ2 = 'permanence' +id_champ1 = 'id_reg_hydro' +id_champ2 = 'id_permanence' +delim1 = zh._get_param(param_table='param_reg_hydro', type_court=False) +delim2 = zh._get_param(param_table='param_permanence', type_court=False) + +col_desc = df.columns[df.columns.str.contains('desc')] +df.drop( + columns=['id','valid', 'date_geom', 'auteur_geom', *col_desc,'type','lb_hab_fr'], + inplace=True, + errors='ignore') + +for i, s in enumerate(lst_site_new): + df.loc[df.id_site==lst_site_old[i], 'id_site'] = s + df.loc[df.id_site==s, ['id_geom_site']] = geomsite.loc[geomsite.id_site==s,'id'].values[0].astype(str) + +df['id_geom_site'] = df.id_geom_site.astype(int) +df[id_champ1] = df[nom_champ1] +df[id_champ2] = df[nom_champ2] +df[id_champ1] = df[id_champ1].replace(delim1.nom.tolist(),delim1.id.tolist()) +df[id_champ2] = df[id_champ2].replace(delim2.nom.tolist(),delim2.id.tolist()) +df['date'] = df[date_champ] +df.in_out = df.in_out == 'entree' + + +df.drop(columns=['id_site',nom_champ1,nom_champ2,date_champ], inplace=True) +lst_idgeom = df.id_geom_site.unique().astype(str) + +# check si data in bdd +sql = 'select exists(select * from {sch}.{tab} where id_geom_site in ({lst_id}));'.format( + sch=sch, tab=tab, lst_id=','.join(lst_idgeom) +) +res = pd.read_sql(sql,con) +if res.exists[0]: + # delete if exist + sql = 'delete from {sch}.{tab} where id_geom_site in ({lst_id});'.format( + sch=sch, tab=tab, lst_id=','.join(lst_idgeom) + ) + pd.read_sql(sql,con) +# insert new line in bdd +col = df.columns[~df.columns.str.contains('auteur')] +colaut = df.columns[df.columns.str.contains('auteur')] +df[col].to_sql( + tab, con, schema=sch, + if_exists='append',index=False,method='multi' +) +# Récupération des nouveaux ids insérés +sql = 'select * from {sch}.{tab} where id_geom_site in ({lst_id});'.format( + sch=sch, tab=tab, lst_id=','.join(lst_idgeom) +) +res = pd.read_sql(sql,con) +df = df.merge(res, how='left', on=[*col]) + +ins_aut = df[['id',*colaut]].copy() +aut['nom_prenom'] = aut.nom + ' ' + aut.prenom +aut.loc[aut.nom_prenom.isna(), 'nom_prenom'] = aut.loc[aut.nom_prenom.isna(), 'nom'] +ins_aut[colaut] = ins_aut[colaut].replace(aut.nom_prenom.tolist(), aut.id.tolist()) +ins_aut.columns = [id_rsiteaut, 'id_auteur'] +ins_aut.to_sql( + rtab_aut, con, schema=sch, + if_exists='append',index=False,method='multi') \ No newline at end of file diff --git a/vrac/Modif_caract_sub.py b/vrac/Modif_caract_sub.py new file mode 100644 index 0000000..835f9d4 --- /dev/null +++ b/vrac/Modif_caract_sub.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- +import pandas as pd +import pyzh +zh = pyzh.zh() + +con = pyzh.con +sch = 'zones_humides' +lst_site_old = ['38RD0146','38RD0147'] +lst_site_new = ['38RD0164','38RD0165'] +df = zh.get_sub(id_site=lst_site_old) +geomsite = zh.get_sitesGeom(id_site=lst_site_new) +aut = pyzh.pers.get_auteur() + +tab = 'r_site_sub' +rtab_aut = 'r_rsitesub_auteur' +id_rsiteaut = 'id_sitesub' +date_champ = 'date_sub' +nom_champ1 = 'Submersion étendue' +nom_champ2 = 'Submersion fréquente' +nom_champ3 = 'origine_sub' +id_champ1 = 'id_freqsub' +id_champ2 = 'id_etendsub' +id_champ3 = 'id_origsub' +delim = zh._get_param(param_table='param_sub',type_table='type_param_sub', type_court=False) +delim1 = delim.loc[delim.type == nom_champ1] +delim2 = delim.loc[delim.type == nom_champ2] + + +df.drop( + columns=['id','valid', 'date_geom', 'auteur_geom', 'desc_param','type','lb_hab_fr'], + inplace=True, + errors='ignore') + +for i, s in enumerate(lst_site_new): + df.loc[df.id_site==lst_site_old[i], 'id_site'] = s + df.loc[df.id_site==s, ['id_geom_site']] = geomsite.loc[geomsite.id_site==s,'id'].values[0].astype(str) + +df['id_geom_site'] = df.id_geom_site.astype(int) +df[id_champ1] = df[nom_champ1] +df[id_champ2] = df[nom_champ2] +df[id_champ3] = df[nom_champ3] +df[id_champ1] = df[id_champ1].replace(delim1.nom.tolist(),delim1.id.tolist()) +df[id_champ2] = df[id_champ2].replace(delim2.nom.tolist(),delim2.id.tolist()) +df['date'] = df[date_champ] + +df.drop(columns=['id_site',nom_champ1,nom_champ2,nom_champ3,date_champ], inplace=True) +lst_idgeom = df.id_geom_site.unique().astype(str) + +# check si data in bdd +sql = 'select exists(select * from {sch}.{tab} where id_geom_site in ({lst_id}));'.format( + sch=sch, tab=tab, lst_id=','.join(lst_idgeom) +) +res = pd.read_sql(sql,con) +if res.exists[0]: + # delete if exist + sql = 'delete from {sch}.{tab} where id_geom_site in ({lst_id});'.format( + sch=sch, tab=tab, lst_id=','.join(lst_idgeom) + ) + pd.read_sql(sql,con) +# insert new line in bdd +col = df.columns[~df.columns.str.contains('auteur')] +colaut = df.columns[df.columns.str.contains('auteur')] +df[col].to_sql( + tab, con, schema=sch, + if_exists='append',index=False,method='multi' +) +# Récupération des nouveaux ids insérés +sql = 'select * from {sch}.{tab} where id_geom_site in ({lst_id});'.format( + sch=sch, tab=tab, lst_id=','.join(lst_idgeom) +) +res = pd.read_sql(sql,con) +df = df.merge(res, how='left', on=[*col]) + +ins_aut = df[['id',*colaut]].copy() +aut['nom_prenom'] = aut.nom + ' ' + aut.prenom +aut.loc[aut.nom_prenom.isna(), 'nom_prenom'] = aut.loc[aut.nom_prenom.isna(), 'nom'] +ins_aut[colaut] = ins_aut[colaut].replace(aut.nom_prenom.tolist(), aut.id.tolist()) +ins_aut.columns = [id_rsiteaut, 'id_auteur'] +ins_aut.to_sql( + rtab_aut, con, schema=sch, + if_exists='append',index=False,method='multi') \ No newline at end of file diff --git a/vrac/Modif_caract_usageprocess.py b/vrac/Modif_caract_usageprocess.py new file mode 100644 index 0000000..f0ed903 --- /dev/null +++ b/vrac/Modif_caract_usageprocess.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- +import pandas as pd +import pyzh +zh = pyzh.zh() + +con = pyzh.con +sch = 'zones_humides' +lst_site_old = ['38RD0146','38RD0147'] +lst_site_new = ['38RD0164','38RD0165'] +df = zh.get_usageprocess(id_site=lst_site_old) +geomsite = zh.get_sitesGeom(id_site=lst_site_new) +aut = pyzh.pers.get_auteur() + +tab = 'r_site_usageprocess' +rtab_aut = 'r_rsiteusage_auteur' +id_rsiteaut = 'id_siteusage' +date_champ = 'date_usage' +nom_champ1 = 'activite_hum' +nom_champ2 = 'impact' +nom_champ3 = 'position' +id_champ1 = 'id_activ_hum' +id_champ2 = 'id_impact' +id_champ3 = 'id_position' +delim1 = zh._get_param(param_table='param_activ_hum', type_court=False) +delim2 = zh._get_param(param_table='param_impact', type_court=False) +delim3 = zh._get_param(param_table='param_position', type_court=False) + +col_desc = df.columns[df.columns.str.contains('desc')] +df.drop( + columns=['id','valid', 'date_geom', 'auteur_geom', *col_desc,'description','type','lb_hab_fr'], + inplace=True, + errors='ignore') + +for i, s in enumerate(lst_site_new): + df.loc[df.id_site==lst_site_old[i], 'id_site'] = s + df.loc[df.id_site==s, ['id_geom_site']] = geomsite.loc[geomsite.id_site==s,'id'].values[0].astype(str) + +df['id_geom_site'] = df.id_geom_site.astype(int) +df[id_champ1] = df[nom_champ1] +df[id_champ2] = df[nom_champ2] +df[id_champ3] = df[nom_champ3] +df[id_champ1] = df[id_champ1].replace(delim1.nom.tolist(),delim1.id.tolist()) +df[id_champ2] = df[id_champ2].replace(delim2.nom.tolist(),delim2.id.tolist()) +df[id_champ3] = df[id_champ3].replace(delim3.nom.tolist(),delim3.id.tolist()) +df['date'] = df[date_champ] + +df.drop(columns=['id_site',nom_champ1,nom_champ2,nom_champ3,date_champ], inplace=True) +lst_idgeom = df.id_geom_site.unique().astype(str) + +# check si data in bdd +sql = 'select exists(select * from {sch}.{tab} where id_geom_site in ({lst_id}));'.format( + sch=sch, tab=tab, lst_id=','.join(lst_idgeom) +) +res = pd.read_sql(sql,con) +if res.exists[0]: + # delete if exist + sql = 'delete from {sch}.{tab} where id_geom_site in ({lst_id});'.format( + sch=sch, tab=tab, lst_id=','.join(lst_idgeom) + ) + pd.read_sql(sql,con) +# insert new line in bdd +col = df.columns[~df.columns.str.contains('auteur')] +colaut = df.columns[df.columns.str.contains('auteur')] +df[col].to_sql( + tab, con, schema=sch, + if_exists='append',index=False,method='multi' +) +# Récupération des nouveaux ids insérés +sql = 'select * from {sch}.{tab} where id_geom_site in ({lst_id});'.format( + sch=sch, tab=tab, lst_id=','.join(lst_idgeom) +) +res = pd.read_sql(sql,con) +df = df.merge(res, how='left', on=[*col]) + +ins_aut = df[['id',*colaut]].copy() +aut['nom_prenom'] = aut.nom + ' ' + aut.prenom +aut.loc[aut.nom_prenom.isna(), 'nom_prenom'] = aut.loc[aut.nom_prenom.isna(), 'nom'] +ins_aut[colaut] = ins_aut[colaut].replace(aut.nom_prenom.tolist(), aut.id.tolist()) +ins_aut.columns = [id_rsiteaut, 'id_auteur'] +ins_aut.to_sql( + rtab_aut, con, schema=sch, + if_exists='append',index=False,method='multi') \ No newline at end of file diff --git a/wfs/__init__.py b/wfs/__init__.py new file mode 100644 index 0000000..939b2fc --- /dev/null +++ b/wfs/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- + +from .wfs import * \ No newline at end of file diff --git a/wfs/__pycache__/__init__.cpython-38.pyc b/wfs/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..5ec9994 Binary files /dev/null and b/wfs/__pycache__/__init__.cpython-38.pyc differ diff --git a/wfs/__pycache__/wfs.cpython-38.pyc b/wfs/__pycache__/wfs.cpython-38.pyc new file mode 100644 index 0000000..3a48f29 Binary files /dev/null and b/wfs/__pycache__/wfs.cpython-38.pyc differ diff --git a/wfs/wfs.py b/wfs/wfs.py new file mode 100644 index 0000000..8ef2f3a --- /dev/null +++ b/wfs/wfs.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- + +from owslib.wfs import WebFeatureService + +def get_wfs(url, layer, bbox=None): + from geopandas import read_file + from requests import Request + wfs = WebFeatureService(url=url) + item = dict(wfs.items())[layer] + crs = item.crsOptions[0].getcode() + params = dict(service='WFS', version=wfs.version, request='GetFeature', + typeName=layer) + q = Request('GET', url, params=params).prepare().url + data = read_file(q, bbox=bbox) + data.set_crs(crs=crs, inplace=True) + if crs != 'EPSG:2154': + data.to_crs(epsg=2154, inplace=True) + return data + +def list_layer(url): + wfs = WebFeatureService(url=url) + lst = list(wfs.contents) + return lst \ No newline at end of file diff --git a/zh.py b/zh.py new file mode 100644 index 0000000..c354749 --- /dev/null +++ b/zh.py @@ -0,0 +1,684 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- +#Nom : : zh.py +#Description : +#Copyright : 2021, CEN38 +#Auteur : Colas Geier +#Version : 1.0 + + + +import pandas as pd +# import pandas_access as mdb +# import numpy as np +# from sqlalchemy.sql.expression import column +from sqlalchemy import create_engine +from geoalchemy2 import Geometry +from .pers.pers import _get_table +from .sites.sites import _get_typ_milieux +from .tools import _get_relation_autor + + +##################################### +### schema personnes ### +##################################### +# class pers: +# def __init__(self): +# from .params import con +# self.schema = 'personnes' +# self.con = con +# # self._get_table = _get_table + +# def get_auteur(self, nom=None, prenom=None): +# sql = 'SELECT * FROM %s.personne'%self.schema +# if nom or prenom : sql = sql + ' WHERE ' +# if nom : +# sql = sql + 'nom IN %(nom)s' +# nom = to_upper(nom) +# if nom and prenom : sql = sql + ' AND ' +# if prenom : +# sql = sql + 'prenom IN %(prenom)s' +# prenom = to_upperfirst(prenom) +# df = pd.read_sql( +# sql = sql, +# con = self.con, +# params = {'nom': to_tuple(nom), 'prenom': to_tuple(prenom) }) +# return df + +# def get_organisme(self, ids=None, nom=None): +# table = 'organisme' +# return _get_table(self.con, self.schema, table, ids=ids, nom=nom) + +##################################### +### schema sites ### +##################################### +# class sites: +# def __init__(self): +# from .params import con +# from .pers import pers +# self.schema = 'sites' +# self.con = con +# self.typ_milieux = self._get_typ_milieux() +# self.typo_sdage = self._get_typo_sdage() +# self.typ_site = self._get_typ_site() +# self.auteur = pers.get_auteur() +# self.organisme = pers.get_organisme() + +# def _get_typ_milieux(self, ids=None, nom=None): +# table = 'type_milieu' +# df = _get_table(self.con, self.schema, table, ids=ids, nom=nom) +# return df + +# def _get_typo_sdage(self, ids=None, nom=None): +# table = 'typo_sdage' +# df = _get_table(self.con, self.schema, table, ids=ids, nom=nom) +# return df + +# def _get_typ_site(self, ids=None, nom=None): +# table = 'type_site' +# df = _get_table(self.con, self.schema, table, ids=ids, nom=nom) +# return df + + # def _merge_orga(self, df, split_cols): + # org = self.organisme + # aut = self.auteur + # df = df.copy() + # for c in split_cols: + # if not isinstance(df[c], int): df[c] = df[c].astype(float) + # df[c].replace(aut.id.tolist(), aut.id_organisme.tolist(), inplace=True) + # df[c].replace(org.id.tolist(), org.nom.tolist(), inplace=True) + # df['organisme'] = None + # for c in split_cols: + # df.loc[df.organisme.isna(), 'organisme'] = df.loc[df['organisme'].isna(), c] + # for c in split_cols: + # comp = df.loc[~df[c].isna(),'organisme'].compare(df.loc[~df[c].isna(), c]) + # if not comp.empty: + # comp['test'] = comp.apply(lambda x: x['other'] in x['self'], axis=1) + # comp = comp[~comp.test] + # if not comp.empty: + # df.loc[comp.index,'organisme'] = comp.self + ' & ' + comp.other + # df.drop(columns=split_cols, inplace=True) + # return df + + # def _merge_author(self, df, col_aut, orga=False): + # # récupération des auteurs + # aut = self.auteur.fillna('') + # aut['nom_prenom'] = (aut['nom'] + ' ' + aut['prenom']).str.strip() + # aut['id'] = aut['id'].astype(str) + # # merge des auteurs + # r_id = df[['id', col_aut]].copy() + # r_idSplit = r_id[col_aut].str.split(' & ', expand=True) + # r_id = r_id.join(r_idSplit) + # cSplit = r_idSplit.shape[1] + # cSplit = list(range(cSplit)) + # if orga: + # # récup des organismes + # org = self._merge_orga(r_id, cSplit) + # r_id[cSplit] = r_id[cSplit].replace(aut['id'].tolist(),aut['nom_prenom'].tolist()) + # r_id = _aggr_cols(r_id,cSplit,' & ') \ + # .rename(columns={'aggreg': 'auteur'}) \ + # .drop(columns=cSplit) + # if orga: + # # merge des organismes + # r_id = pd.merge(r_id,org, on=['id', col_aut]) + # df = pd.merge(df,r_id, on=['id', col_aut]) \ + # .drop(columns=[col_aut]) + # return df + + # def _merge_relation(self, df, table, schema, id=None, left_id=None,right_id=None): + # con = self.con + # if id: + # params = {id: df[id].tolist() } + # elif left_id and right_id: + # params = {right_id: df[left_id].tolist() } + # mrg = _get_table(con, schema, table, params_col=params) + # if table == 'r_sites_auteur' or table == 'r_geomsites_auteur': + # mrg = mrg[[right_id,'id_auteur']].groupby( + # [right_id])['id_auteur'].apply(lambda x: ' & '.join(x.astype(str))) + # mrg = pd.DataFrame(data=mrg) + # if id: + # df = pd.merge(df,mrg, how='left', on=id) + # elif left_id and right_id: + # df = pd.merge(df,mrg, how='left', left_on=left_id, right_on=right_id) + # return df + + + # def get_sitesInfos(self, ids=None, nom=None, columns=None, with_nameOrga=False, details=False, params_col={}): + # drop = [] + # table = 'sites' + # df = _get_table(self.con, self.schema, table, ids=ids, nom=nom, cols=columns, params_col=params_col) + # # récupération des auteurs + # if 'id_auteur' in df.columns: + # df.drop(columns='id_auteur', inplace=True) + # df = _merge_relation(df=df,table='r_sites_auteur',schema=self.schema, left_id='id',right_id='id_site') + # df = _merge_author(df=df, col_aut='id_auteur', orga=with_nameOrga) + # # merge type_site + # if 'id_type_site' in df.columns: + # df = pd.merge(df, self.typ_site, how='left', left_on='id_type_site', right_on='id', suffixes=('','_y') ) \ + # .drop(columns=['id_type_site', 'id_y']) \ + # .rename(columns={'nom_y': 'type_site', 'description': 'desc_type_site'}) + # drop += ['desc_type_site'] + # # merge typo_sdage + # if 'id_typo_sdage' in df.columns: + # df = pd.merge(df, self.typo_sdage, how='left', left_on='id_typo_sdage', right_on='id', suffixes=('','_y') ) \ + # .drop(columns=['id_typo_sdage', 'id_y']) \ + # .rename(columns={'nom_y': 'typo_sdage', 'description': 'desc_typo_sdage'}) + # drop += ['desc_typo_sdage'] + # # merge type_milieu + # if 'id_type_milieu' in df.columns: + # df = pd.merge(df, self.typ_milieux, how='left', left_on='id_type_milieu', right_on='id', suffixes=('','_y') ) \ + # .drop(columns=['id_type_milieu', 'id_y']) \ + # .rename(columns={'nom_y': 'type_milieu', 'description': 'desc_type_milieu', 'nom_court': 'nom_court_milieu'}) + # drop += ['desc_type_milieu', 'nom_court_milieu'] + + # if not details: + # df.drop(columns=drop, inplace=True) + + # return df.sort_values('id') + + # def get_sitesGeom(self, id_site=None, nom_site=None, columns=None, last_update=False, with_nameOrga=False, params_col={}): + # # from shapely.wkb import loads + # # import geopandas as gpd # set_geometry + + # if columns: + # if not isinstance(columns, list): columns = [columns] + # if 'id' not in columns: columns.insert(0,'id') + # if 'id_site' not in columns: columns.insert(1,'id_site') + # if 'geom' not in columns: columns.insert(2,'geom') + + # table = 'sites' + # df = _get_table(self.con, self.schema, table, ids=id_site, nom=nom_site, cols='id', params_col=params_col) + # idSite = df.id.tolist() + # table = 'r_sites_geom' + # df = _get_table(self.con, self.schema, table, params_col={'id_site':idSite}, cols=columns) + # if last_update: + # df.drop_duplicates(subset=['id_site'], keep='last', inplace=True) + # df.reset_index(inplace=True, drop=True) + + + # # df = _set_geom(df) + # # df['geom'] = [(loads(geom, hex=True)) for geom in df['geom']] + # # df = df.set_geometry('geom', crs='EPSG:2154') + # # merge auteur + # if 'id_auteur' in df.columns: + # df.drop(columns='id_auteur', inplace=True) + # df = _merge_relation(df=df,table='r_geomsites_auteur',schema=self.schema, left_id='id',right_id='id_geom_site') + # df = _merge_author(df=df, col_aut='id_auteur', orga=with_nameOrga) + + # return df + + + +##################################### +### schema zh ### +##################################### +class zh(): + def __init__(self): + from .params import con + from .tools import _get_relation_tab + self.schema = 'zones_humides' + self.con = con + self.typ_milieux = _get_typ_milieux(nom='Tourbières et marais') + self.id_milieux = self.typ_milieux.id.values[0] + self._get_relation_tab = _get_relation_tab + self.lst_tab = con.dialect.get_table_names(con,schema=self.schema) + self.columns_rSiteFcts = con.dialect.get_columns(con,schema=self.schema,table_name='r_site_fctecosociopatri') + self.columns_r_SiteCon = con.dialect.get_columns(con,schema=self.schema,table_name='r_site_type_connect') + self.columns_rSiteCritDelim = con.dialect.get_columns(con,schema=self.schema,table_name='r_site_critdelim') + self.columns_r_SiteHabs = con.dialect.get_columns(con,schema=self.schema,table_name='r_site_habitat') + self.columns_rSiteRegHyd = con.dialect.get_columns(con,schema=self.schema,table_name='r_site_reghydro') + self.columns_r_SiteSub = con.dialect.get_columns(con,schema=self.schema,table_name='r_site_sub') + self.columns_r_SiteUsgPrss = con.dialect.get_columns(con,schema=self.schema,table_name='r_site_usageprocess') + + + def _get_param(self, param_table, type_table=None, type_court=True): + if type_table: + typ = _get_table(self.con, self.schema, table=type_table) + par = _get_table(self.con, self.schema, table=param_table, params_col={'id_type':typ.id.tolist()}) + df = pd.merge(par, typ, left_on='id_type', right_on='id', how='left', suffixes=(None, '_typ')) \ + .drop(columns=['id_type','id_typ']) + if 'description_typ' in df.columns: del df['description_typ'] + if type_court: df = df.drop(columns=['nom_typ']).rename(columns={'nom_court_typ':'type'}) + else : df = df.drop(columns=['nom_court_typ'],errors='ignore').rename(columns={'nom_typ':'type'}) + df = df.set_index(['id', 'type']).reset_index() + else: + df = _get_table(self.con, self.schema, table=param_table) + + return df + + + # def _get_relation_tab(self, tab, id_site=None, nom_site=None, last_update=False, geom=False,params_col={}): + # table = 'sites' + # dfSG = get_sitesGeom(columns='date', id_site=id_site, nom_site=nom_site, last_update=last_update,params_col=params_col) + # if not geom and not dfSG.empty: + # dfSG.drop('geom',1,inplace=True) + # ids = dfSG.id.tolist() + # table = tab + + # if ids : + # df = _get_table(self.con, self.schema, table, params_col={'id_geom_site':ids}) + # # if not df.empty: + # df = pd.merge(dfSG,df, how='left', left_on='id', right_on='id_geom_site', suffixes=('_x', None)) \ + # .drop(['id_x','id_geom_site'],1) \ + # .set_index('id').reset_index() + # return df + # else: + # print('PAS de géometries de sites sélectionnées ...') + + + def get_delim(self, id_site=None, nom_site=None, last_update=True, geom=False, + nom_type_court=True,statut='actif'): + + table = 'r_site_critdelim' + df = self._get_relation_tab( + schema=self.schema,tab=table,id_site=id_site,nom_site=nom_site,last_update=last_update,geom=geom, + params_col={'id_type_milieu':self.id_milieux.astype(str)}, statut=statut) + dic = self._get_param(type_table='type_param_delim_fct', param_table='param_delim_fct', type_court=nom_type_court) + + if not df.empty: + df = _get_relation_autor(df, relation_tab='r_rsitedelim_auteur', schema=self.schema, + id_df='id', id_relation='id_sitedelim', id_rela_auth='id_auteur' ) + + df = pd.merge(df,dic, how='left', left_on='id_crit_delim', right_on='id', suffixes=(None,'_y')) \ + .drop(['id_y','id_crit_delim'],1) \ + .rename(columns={'description_y':'desc_param', 'nom_court':'nom_court_crit','nom':'nom_crit'}) \ + .sort_values('id_site') + + if df.nom_court_crit.isnull().sum() == df.shape[0] : del df['nom_court_crit'] + # typ = df.type.unique() + # x = {} + # for t in typ: + # x[t] = df[df.type == t] + # x[t] = x[t].rename(columns={'nom': t}) \ + # .reset_index(drop=True) + + return df + + + def get_fct(self, id_site=None, nom_site=None, last_update=True, geom=False, nom_type_court=True,statut='actif'): + + table = 'r_site_fctecosociopatri' + df = self._get_relation_tab(schema=self.schema,tab=table,id_site=id_site,nom_site=nom_site,last_update=last_update,geom=geom, + params_col={'id_type_milieu':self.id_milieux.astype(str)}, statut=statut) + dic = self._get_param(type_table='type_param_fct', param_table='param_fct_eco_socio_patri', type_court=nom_type_court) + + if not df.empty: + df = _get_relation_autor(df, relation_tab='r_rsitefct_auteur', schema=self.schema, + id_df='id', id_relation='id_sitefct', id_rela_auth='id_auteur' ) + + df = pd.merge(df,dic, how='left', left_on='id_fct', right_on='id', suffixes=(None,'_y')) \ + .drop(['id_y','id_fct'],1) \ + .rename(columns={'description_y':'desc_param', 'nom_court':'nom_court_fct','nom':'nom_fct'}) \ + .sort_values('id_site') + + if df.nom_court_fct.isnull().sum() == df.shape[0] : del df['nom_court_fct'] + + return df + + + def get_connex(self, id_site=None, nom_site=None, last_update=True, geom=False,statut='actif'): + + table = 'r_site_type_connect' + df = self._get_relation_tab(schema=self.schema,tab=table,id_site=id_site,nom_site=nom_site,last_update=last_update,geom=geom, + params_col={'id_type_milieu':self.id_milieux.astype(str)}, statut=statut) + dic = self._get_param(param_table='param_type_connect') + + if not df.empty: + df = _get_relation_autor(df, relation_tab='r_rsiteconnect_auteur', schema=self.schema, + id_df='id', id_relation='id_siteconnect', id_rela_auth='id_auteur' ) + df = pd.merge(df,dic, how='left', left_on='id_param_connect', right_on='id', suffixes=(None,'_y')) \ + .drop(['id_y','id_param_connect'],1) \ + .rename(columns={'description_y':'desc_param', 'nom':'connexion'}) \ + .sort_values('id_site') + + return df + + + def get_sub(self, id_site=None, nom_site=None, last_update=True, geom=False,statut='actif'): + + table = 'r_site_sub' + df = self._get_relation_tab(schema=self.schema,tab=table,id_site=id_site,nom_site=nom_site,last_update=last_update,geom=geom, + params_col={'id_type_milieu':self.id_milieux.astype(str)}, statut=statut) + dic = self._get_param(type_table='type_param_sub', param_table='param_sub', type_court=False) + d1 = dic[dic.type == 'Submersion étendue'] + d2 = dic[dic.type == 'Submersion fréquente'] + + if not df.empty: + df = _get_relation_autor(df, relation_tab='r_rsitesub_auteur', schema=self.schema, + id_df='id', id_relation='id_sitesub', id_rela_auth='id_auteur' ) + + df = pd.merge(df,d1, how='left', left_on='id_etendsub', right_on='id', suffixes=(None,'_y')) \ + .drop(['id_y','id_etendsub', 'type'],1) \ + .rename(columns={'description':'desc_param_etend', 'nom':'Submersion étendue'}) + + df = pd.merge(df,d2, how='left', left_on='id_freqsub', right_on='id', suffixes=(None,'_y')) \ + .drop(['id_y','id_freqsub', 'type'],1) \ + .rename(columns={'description':'desc_param_freq', 'nom':'Submersion fréquente'}) \ + .sort_values('id_site') + + df.rename(columns={'id_origsub': 'origine_sub'}, inplace=True) + + if df['desc_param_etend'].isnull().sum() == df.shape[0] : del df['desc_param_etend'] + if df['desc_param_freq'].isnull().sum() == df.shape[0] : del df['desc_param_freq'] + + return df + + + def get_usageprocess(self, id_site=None, nom_site=None, last_update=True, geom=False,statut='actif'): + + table = 'r_site_usageprocess' + df = self._get_relation_tab(schema=self.schema,tab=table,id_site=id_site,nom_site=nom_site,last_update=last_update,geom=geom, + params_col={'id_type_milieu':self.id_milieux.astype(str)}, statut=statut) + dic1 = self._get_param(param_table='param_activ_hum') + dic2 = self._get_param(param_table='param_position') + dic3 = self._get_param(param_table='param_impact') + + if not df.empty: + df = _get_relation_autor(df, relation_tab='r_rsiteusage_auteur', schema=self.schema, + id_df='id', id_relation='id_siteusage', id_rela_auth='id_auteur' ) + + df = pd.merge(df,dic1, how='left', left_on='id_activ_hum', right_on='id', suffixes=(None,'_y')) \ + .drop(['id_y','id_activ_hum'],1) \ + .rename(columns={'description':'desc_param_usag', 'nom':'activite_hum'}) + df = pd.merge(df,dic2, how='left', left_on='id_position', right_on='id', suffixes=(None,'_y')) \ + .drop(['id_y','id_position'],1) \ + .rename(columns={'description':'desc_param_pos', 'nom':'position'}) \ + .sort_values('id_site') + df = pd.merge(df,dic3, how='left', left_on='id_impact', right_on='id', suffixes=(None,'_y')) \ + .drop(['id_y','id_impact'],1) \ + .rename(columns={'description':'desc_param_imp', 'nom':'impact'}) \ + .sort_values('id_site') + + return df + + + def _get_r_toponymie(self, ids=None): + table = 'r_toponymie' + df = _get_table(self.con, self.schema, table=table, ids=ids) + dic1 = self._get_param(param_table='liste_table_topohydro') + if not df.empty: + df = pd.merge(df,dic1, how='left', left_on='id_orig', right_on='id', suffixes=(None,'_y')) \ + .drop(['id_y','id_orig'],1) + n_tab = df.nom_table.unique() + for tab in n_tab: + iids = df.loc[df.nom_table == tab, 'id_topo'].to_list() + if tab == 'orig_hydro': dic = _get_table(self.con, self.schema, table='orig_hydro', ids=iids) + if tab == 'troncon_hydro': dic = ref_hydro.get_troncon(cols=['id','nom'], ids=iids) + df.loc[df.nom_table == tab, 'id_topo'] = df.loc[df.nom_table == tab, 'id_topo'].replace(dic.id.to_list(),dic.nom.to_list()) + if tab == 'troncon_hydro': df = pd.merge(df, dic, how='left', left_on='id_topo', right_on='nom', suffixes=(None,'_y')) \ + .drop(columns=['id_y', 'nom']) + df.rename(columns={'id_topo':'toponymie'}) + + return df + + + def get_regHydro(self, id_site=None, nom_site=None, last_update=True, geom=False,statut='actif'): + + table = 'r_site_reghydro' + df = self._get_relation_tab(schema=self.schema,tab=table,id_site=id_site,nom_site=nom_site,last_update=last_update,geom=geom, + params_col={'id_type_milieu':self.id_milieux.astype(str)}, statut=statut) + dic1 = self._get_param(param_table='param_reg_hydro') + dic2 = self._get_param(param_table='param_permanence') + + if not df.empty: + df = _get_relation_autor(df, relation_tab='r_rsitehydro_auteur', schema=self.schema, + id_df='id', id_relation='id_sitehydro', id_rela_auth='id_auteur' ) + + # dic3 = self._get_r_toponymie(ids=df.rmq_toponymie.unique().tolist()) + df.in_out = df.in_out.replace([True,False],['entree','sortie']) + df = pd.merge(df,dic1, how='left', left_on='id_reg_hydro', right_on='id', suffixes=(None,'_y')) \ + .drop(['id_y','id_reg_hydro'],1) \ + .rename(columns={'description':'desc_param_regHydri', 'nom':'regime_hydri'}) + + if df.id_permanence.isna().all() : + df.rename(columns={'id_permanence':'permanence'}, inplace=True) + df['desc_param_perm'] = None + else: + df = pd.merge(df,dic2, how='left', left_on='id_permanence', right_on='id', suffixes=(None,'_y')) \ + .drop(['id_y','id_permanence'],1) \ + .rename(columns={'description':'desc_param_perm', 'nom':'permanence'}) + + # df = pd.merge(df,dic3, left_on='rmq_toponymie', right_on='id', suffixes=(None,'_y')) \ + # .drop(['id_y','rmq_toponymie'],1) \ + # .rename(columns={'description':'desc_topo'}) \ + # .sort_values('id_site') + + return df + + + def get_habitat(self, id_site=None, nom_site=None, last_update=True, geom=False,statut='actif'): + + table = 'r_site_habitat' + df = self._get_relation_tab(schema=self.schema,tab=table,id_site=id_site,nom_site=nom_site,last_update=last_update,geom=geom, + params_col={'id_type_milieu':self.id_milieux.astype(str)}, statut=statut) + + if not df.empty: + df = _get_relation_autor(df, relation_tab='r_rsitehab_auteur', schema=self.schema, + id_df='id', id_relation='id_sitehab', id_rela_auth='id_auteur' ) + + ids = df[~df.id_cb.isna()].id_cb.unique().tolist() + dic = ref_hab().get_CB(ids=ids,cols=['id','lb_hab_fr']) + df = pd.merge(df,dic, how='left', left_on='id_cb', right_on='id', suffixes=(None,'_y')) \ + .drop(['id_y'], 1) \ + .rename(columns={'id_cb':'code_cb'}) \ + .sort_values('id_site') + + return df + + + def get_sitesInfos(self, id_site=None, nom_site=None, columns=None, with_nameOrga=False, statut='actif'): + from .sites.sites import get_sitesInfos + + df = get_sitesInfos(ids=id_site, nom=nom_site, columns=columns, with_nameOrga=with_nameOrga, + params_col={'id_type_milieu':self.id_milieux.astype(str)}, statut=statut) + + return df + + + def get_sitesGeom(self, id_site=None, nom_site=None, columns=None, last_update=True, with_nameOrga=False, + params_col={}, statut='actif'): + from .sites.sites import get_sitesGeom + + if params_col: + params_col = {**params_col, 'id_type_milieu':self.id_milieux.astype(str) } + + df = get_sitesGeom(id_site=id_site, nom_site=nom_site, columns=columns, + with_nameOrga=with_nameOrga,last_update=last_update, + params_col=params_col , statut=statut) + + drop_cols = ['link_pdf', 'rmq_fct_majeur', + 'rmq_interet_patri', 'rmq_bilan_menace', 'rmq_orient_act', + 'rmq_usage_process'] + cols = df.columns + c = cols[cols.isin(drop_cols)] + if not c.empty: + df.drop(columns=c, inplace=True) + + return df + + + def get_evaluation(self, id_site=None, nom_site=None, columns=None, last_update=True, with_nameOrga=False, + params_col={}, statut='actif'): + from .sites.sites import get_sitesGeom + + if params_col: + params_col = {**params_col, 'id_type_milieu':self.id_milieux.astype(str) } + + df = get_sitesGeom(id_site=id_site, nom_site=nom_site, columns=columns, + with_nameOrga=with_nameOrga,last_update=last_update, + params_col=params_col, statut=statut) + + df.drop(columns=['geom'], inplace=True) + + return df + + + def get_bilan(self, code_site=None, nom_site=None, statut='actif',last_update=True): + ''' + :code_site: list,str. Code du site de la zh. + :nom_site: list,str. Nom du site de la zh. + ''' + info = self.get_sitesInfos(id_site=code_site, nom_site=nom_site,statut=statut) + CB = self.get_habitat(id_site=code_site, nom_site=nom_site,last_update=last_update,statut=statut) + delim = self.get_delim(id_site=code_site, nom_site=nom_site,last_update=last_update,statut=statut) + desc = self.get_usageprocess(id_site=code_site, nom_site=nom_site,last_update=last_update,statut=statut) + rghyd = self.get_regHydro(id_site=code_site, nom_site=nom_site,last_update=last_update,statut=statut) + subm = self.get_sub(id_site=code_site, nom_site=nom_site,last_update=last_update,statut=statut) + conn = self.get_connex(id_site=code_site, nom_site=nom_site,last_update=last_update,statut=statut) + fct = self.get_fct(id_site=code_site, nom_site=nom_site,last_update=last_update,statut=statut) + evall = self.get_evaluation(id_site=code_site, nom_site=nom_site,last_update=last_update,statut=statut) + + print('subm : {}'.format(subm)) + print('conn : {}'.format(conn)) + if subm.empty: + sub_con = conn.rename(columns={ + 'description': 'desc_connex', + 'valid': 'valid_cnx', + }) + elif conn.empty: + sub_con = subm.rename(columns={'valid': 'valid_sub',}) + else: + sub_con = pd.merge(subm, conn, how='outer', on=['id', 'id_site', 'date_geom','auteur_geom']) \ + .rename(columns={ + 'description': 'desc_connex', + 'valid_x': 'valid_sub', + 'valid_y': 'valid_cnx', + }) + fctmt = { + 'entree_eau': rghyd[rghyd.in_out == 'entree'].drop(columns=['in_out']), + 'sortie_eau': rghyd[rghyd.in_out == 'sortie'].drop(columns=['in_out']), + 'sub_connex': sub_con, + } + lst_df = { + 'infos':info, + 'corine_biotope': CB, + 'delimitation': delim, + 'description': desc, + 'fonctionnement': fctmt, + 'fonction': fct, + 'evaluation': evall} + for key in lst_df: + if isinstance(lst_df[key], pd.DataFrame): lst_df[key].name = key + if isinstance(lst_df[key], dict): + for d in lst_df[key]: + lst_df[key][d].name = d + lst_df[key]['title'] = key + return lst_df + +##################################### +### schema ref_habitats ### +##################################### +class ref_hab: + def __init__(self): + from .params import con + self.schema = 'ref_habitats' + self.con = con + + def get_CB(self, ids=None, cols=None, params_col={}): + table = 'corine_biotope' + df = _get_table(self.con, self.schema, table=table, ids=ids, cols=cols, params_col=params_col) + return df + + +##################################### +### schema ref_hydro ### +##################################### +class ref_hydro: + def __init__(self): + from .params import con + self.schema = 'ref_hydro' + self.con = con + + def get_troncon(self, ids=None, cols=None, params_col={}): + + table = 'troncon_hydro' + df = _get_table(self.con, self.schema, table=table, ids=ids, cols=cols, params_col=params_col) + # df = _set_geom(df) + + return df + + def get_coursEau(self, ids=None, cols=None, params_col={}): + + table = 'cours_eau' + df = _get_table(self.con, self.schema, table=table, ids=ids, cols=cols, params_col=params_col) + # df = _set_geom(df) + + return df + + def get_masseEau(self, ids=None, cols=None, params_col={}): + + table = 'masse_eau' + df = _get_table(self.con, self.schema, table=table, ids=ids, cols=cols, params_col=params_col) + # df = _set_geom(df) + + return df + + def get_planEau(self, ids=None, cols=None, params_col={}): + + table = 'plan_eau' + df = _get_table(self.con, self.schema, table=table, ids=ids, cols=cols, params_col=params_col) + # df = _set_geom(df) + + return df + + def get_ssbv(self, ids=None, cols=None, params_col={}): + + table = 'ssbv' + df = _get_table(self.con, self.schema, table=table, ids=ids, cols=cols, params_col=params_col) + # df = _set_geom(df) + + return df + + + + + +##################################### +### Update ### +##################################### +def update_to_sql(df, con, table_name, schema_name, key_name): + a = [] + b = [] + table = table_name + schema = schema_name + primary_key = key_name + if isinstance(primary_key, str): primary_key = [primary_key] + for col in df.columns: + if col in primary_key: + b.append("t.{col}=f.{col}".format(col=col)) + else: + a.append("{col}=t.{col}".format(col=col)) + df.to_sql( + name = 'temp_table', + con = con, + schema = schema, + if_exists = 'replace', + index = False, + method = 'multi' + ) + update_stmt_1 = "UPDATE {sch}.{final_table} f".format(sch=schema,final_table=table) + update_stmt_2 = " FROM {sch}.temp_table t".format(sch=schema) + update_stmt_6 = " WHERE %s"%' AND '.join(b) + update_stmt_3 = " SET " + update_stmt_4 = ", ".join(a) + update_stmt_5 = update_stmt_1 + update_stmt_3 + update_stmt_4 + update_stmt_2 + update_stmt_6 + ";" + drop_stmt = "DROP TABLE {sch}.temp_table ;".format(sch=schema) + with con.begin() as cnx: + cnx.execute(update_stmt_5) + cnx.execute(drop_stmt) + return print('END update') + + +# [SQL: INSERT INTO zones_humides.r_site_reghydro (id, id_geom_site, id_reg_hydro, id_permanence, rmq_toponymie, in_out) +# VALUES (%(id)s, %(id_geom_site)s, %(id_reg_hydro)s, %(id_permanence)s, %(rmq_toponymie)s, %(in_out)s)] +# [parameters: ({'id': 0, 'id_geom_site': 5, 'id_reg_hydro': '0', 'id_permanence': '1', 'rmq_toponymie': '', 'in_out': True}, +# {'id': 1, 'id_geom_site': 5, 'id_reg_hydro': '1', 'id_permanence': '1', 'rmq_toponymie': '', 'in_out': False}, +# {'id': 2, 'id_geom_site': 6, 'id_reg_hydro': '0', 'id_permanence': '1', 'rmq_toponymie': '', 'in_out': True}, +# {'id': 3, 'id_geom_site': 6, 'id_reg_hydro': '1', 'id_permanence': '1', 'rmq_toponymie': '', 'in_out': False}, +# {'id': 4, 'id_geom_site': 7, 'id_reg_hydro': '2', 'id_permanence': '2', 'rmq_toponymie': 'plusieurs petites sources dans versant', 'in_out': True}, +# {'id': 5, 'id_geom_site': 7, 'id_reg_hydro': '1', 'id_permanence': '2', 'rmq_toponymie': 'longe la route D209a', 'in_out': False}, +# {'id': 6, 'id_geom_site': 8, 'id_reg_hydro': '0', 'id_permanence': '2', 'rmq_toponymie': '', 'in_out': True}, +# {'id': 7, 'id_geom_site': 8, 'id_reg_hydro': '3', 'id_permanence': None, 'rmq_toponymie': '', 'in_out': False} +# ... displaying 10 of 5779 total bound parameter sets ... +# {'id': 5777, 'id_geom_site': 1951, 'id_reg_hydro': '0', 'id_permanence': None, 'rmq_toponymie': '', 'in_out': True}, +# {'id': 5778, 'id_geom_site': 1951, 'id_reg_hydro': '3', 'id_permanence': None, 'rmq_toponymie': '', 'in_out': False})]