From 5291b3b83c90fff9c76f2dc619600104b43f1ba6 Mon Sep 17 00:00:00 2001 From: Colas Geier Date: Tue, 25 Feb 2025 16:52:00 +0100 Subject: [PATCH] update code --- 0_FONCIER/foncier_insert_cadastre_V4.py | 11 +- 3_AZALEE/azaleeTOgeonature.py | 30 +- 3_AZALEE/create_view_ps.py | 11 +- 3_AZALEE/create_view_zh3.py | 86 +++-- 3_AZALEE/insert_ps.py | 25 +- 3_AZALEE/insert_zh.py | 30 +- .../EXPORT/flore_export_sinp_with_metadata.py | 353 +++++++++--------- 5_GEONATURE/insert_liste_alerte.py | 102 ++++- 5_GEONATURE/pivot_bdc_status_v2.py | 212 +++-------- manage_bdd.py | 2 +- 10 files changed, 442 insertions(+), 420 deletions(-) diff --git a/0_FONCIER/foncier_insert_cadastre_V4.py b/0_FONCIER/foncier_insert_cadastre_V4.py index ad5f800..65bdd21 100644 --- a/0_FONCIER/foncier_insert_cadastre_V4.py +++ b/0_FONCIER/foncier_insert_cadastre_V4.py @@ -272,7 +272,8 @@ def _insert_parcelle1(schema='38_202207',list_parid=None): ELSE ltrim(dnuplam, '0') :: INTEGER END AS dnuplam, -- Numéro Parcelle mère - type_filiation AS type -- Type de filiation + type_filiation AS "type", -- Type de filiation + jdatat FROM "{sch}".parcelle a JOIN "{sch}".geo_parcelle ON geo_parcelle.geo_parcelle = a.parcelle ) @@ -330,7 +331,8 @@ def _insert_parcelle2(schema='38_202207',list_parid=None): THEN NULL :: INTEGER ELSE ltrim(a.dnuplam, '0') :: INTEGER END AS dnuplam, -- Numéro Parcelle mère - a.type_filiation AS "type" -- Type de filiation + a.type_filiation AS "type", -- Type de filiation + a.jdatat FROM "{sch}".parcelle a ) SELECT t1.par_id, @@ -348,7 +350,8 @@ def _insert_parcelle2(schema='38_202207',list_parid=None): t1.ccoprem, t1.ccosecm, t1.dnuplam, - t1.type + t1.type, + t1.jdatat FROM t1 LEFT JOIN "{sch}".parcelle_info b ON t1.parcelle = b.geo_parcelle -- les parcelles sans geom LEFT JOIN cadastre.parcelles_cen ON t1.par_id = parcelles_cen.par_id @@ -586,7 +589,7 @@ def _insert_lot1(schema='38_202207'): null AS dnulot, -- Numéro du lot dcntpa AS dcntlo-- Contenance cadastrale (m²) FROM "{sch}".parcelle - JOIN "{sch}".geo_parcelle ON parcelle.parcelle = geo_parcelle.geo_parcelle -- on ne garde que les les parcelles dont on a la géométrie + JOIN "{sch}".geo_parcelle ON parcelle.parcelle = geo_parcelle.geo_parcelle -- on ne garde que les parcelles dont on a la géométrie WHERE ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla IN (SELECT ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||trim(dnulot) FROM "{sch}".suf); -- toutes les parcelles dont dnulot est NULL '''.format( diff --git a/3_AZALEE/azaleeTOgeonature.py b/3_AZALEE/azaleeTOgeonature.py index ffd956f..c5172d3 100644 --- a/3_AZALEE/azaleeTOgeonature.py +++ b/3_AZALEE/azaleeTOgeonature.py @@ -285,7 +285,8 @@ def recup_id_role(author): # A finir ! azalee_auth = pers.get_auteur2().sort_index()#.replace({' ':' '},regex=True) azalee_auth = azalee_auth[azalee_auth.nom_prenom.isin(adapt_auth)].replace({'Inconnu':'Autre'}) # azalee_auth.nom_prenom.replace({'Inconnu':'Autre'},regex=True,inplace=True) - t_roles = pd.merge(get_t_roles().reset_index(),azalee_auth, how='inner',left_on=['nom_role','prenom_role','nom_organisme'],right_on=['nom','prenom','organisme']) + tr = get_t_roles().reset_index().replace({'':None}) + t_roles = pd.merge(tr,azalee_auth, how='inner',left_on=['nom_role','prenom_role','nom_organisme'],right_on=['nom','prenom','organisme']) dict_role = dict(zip(t_roles.nom_prenom,t_roles.id_role)) return author.replace({' \(Inconnu\)':'',' ':' '},regex=True).str.strip().replace(dict_role) @@ -1098,16 +1099,43 @@ def OTHERINV_to_tref(db_file): ) +def trunc_table(table,cascade=False): + """ + Tronque la table pr_zh.table + + Parameters + ---------- + table : str + Nom de la table à tronquer + cascade : bool, optional + Si True, la suppression se fait avec l'option CASCADE. Cela signifie que + les clés étrangères pointant vers cette table seront également supprimées + (par exemple, si vous supprimez une zone humide, vous supprimez + automatiquement les données de suivis liées à cette zone). Par défaut, + cette option est à False. + """ + cascade = 'CASCADE;' if cascade else ';' + sql = 'TRUNCATE pr_zh.%s %s'%(table,cascade) + with con_gn.begin() as cnx: + cnx.execute(sql) if __name__ == "__main__": + # TRUNCATE TABLE + # trunc_table('t_zh',cascade=True) + # trunc_table('cor_zh_area') + # trunc_table('t_reference') + + from pycen.geonature import pr_zh t_zh = pr_zh.t_zh() + drop_cols = ['auteur_geom','date_geom','type_milieu','type_site',] DF = zh.v_zoneshumides() DF.rename(columns=DICT_TZH,inplace=True) DF.drop(columns=drop_cols,inplace=True) + df = DF.copy() df = DF[~DF.code.isin(t_zh.code)].copy() migrate_to_gnZH(df) diff --git a/3_AZALEE/create_view_ps.py b/3_AZALEE/create_view_ps.py index 91fc49c..667d521 100644 --- a/3_AZALEE/create_view_ps.py +++ b/3_AZALEE/create_view_ps.py @@ -6,6 +6,7 @@ from sqlalchemy.engine import URL from datetime import datetime as dt import pandas as pd import geopandas as gpd +from pycen import con # Parametres bdd @@ -13,7 +14,7 @@ user = 'cen_admin' pwd = '#CEN38@venir' adr = '91.134.194.221' port = '5432' -base = 'azalee' +base = 'azalee_20240731' url = URL.create('postgresql+psycopg2', username=user, @@ -21,7 +22,7 @@ url = URL.create('postgresql+psycopg2', host=adr, database=base, ) -con = create_engine(url) +# con = create_engine(url) drop_v_ps = ''' DROP VIEW IF EXISTS ps.v_pelouseseches_all CASCADE; @@ -165,10 +166,10 @@ FROM crosstab( a.id_geom_site::bigint, a.date, (SELECT MAX(date) FROM ps.r_site_param WHERE id_geom_site = a.id_geom_site ) date_max, - c.auteur, + STRING_AGG(distinct c.auteur,', '), a.taux, a2.nom, - a1.description + STRING_AGG(a1.description,', ') FROM ps.r_site_param a JOIN (ps.param a1 JOIN ps.type_param a2 ON a1.id_type = a2.id @@ -176,7 +177,7 @@ FROM crosstab( JOIN auteur c ON c.id_siteparam = a.id --WHERE -- a.id_geom_site in (52716) - --GROUP BY 1 + GROUP BY 1,2,7,3,6 ORDER BY 1,2,7,3 DESC $$, $$SELECT nom FROM ps.type_param ORDER BY nom;$$ diff --git a/3_AZALEE/create_view_zh3.py b/3_AZALEE/create_view_zh3.py index 60c603d..8f83975 100644 --- a/3_AZALEE/create_view_zh3.py +++ b/3_AZALEE/create_view_zh3.py @@ -5,6 +5,22 @@ from sqlalchemy import text from datetime import datetime as dt import geopandas as gpd from pycen import con +# Parametres bdd +from sqlalchemy import create_engine, text +from sqlalchemy.engine import URL +user = 'cen_admin' +pwd = '#CEN38@venir' +adr = '91.134.194.221' +port = '5432' +base = 'azalee_20240731' + +url = URL.create('postgresql+psycopg2', + username=user, + password=pwd, + host=adr, + database=base, +) +con = create_engine(url) drop_v_zh = 'DROP VIEW IF EXISTS zones_humides.v_zoneshumides CASCADE;' with con.begin() as cnx: @@ -15,7 +31,7 @@ v_zh_hab = """ DROP VIEW IF EXISTS zones_humides.v_zh_hab; CREATE OR REPLACE VIEW zones_humides.v_zh_hab AS -WITH auteur AS ( +WITH author AS ( SELECT DISTINCT ON (id_sitehab) c.id_sitehab, string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur @@ -23,21 +39,28 @@ WITH auteur AS ( JOIN personnes.v_personne c1 ON c1.id = c.id_auteur GROUP BY c.id_sitehab ORDER BY 1 +), + d_max as ( + SELECT distinct on (id_site) + a.id_site, + c.auteur, + a."date" + FROM zones_humides.r_site_habitat a + JOIN author c ON c.id_sitehab = a.id + WHERE a."valid" + ORDER BY a.id_site, "date" DESC ) -SELECT DISTINCT ON (a.id_site) - --a.id_geom_site, - a.id_site, - c.auteur, - --MAX(a."date") "date", - a."date", - string_agg(a.id_cb,';' order by a.id_cb) code_cb, - string_agg(b.lb_hab_fr,';' order by a.id_cb) lib_cb -FROM zones_humides.r_site_habitat a +SELECT --distinct on (a.id_cb) + d.id_site, + d.auteur, + d."date", + string_agg(a.id_cb,';' order by a.id_cb asc) code_cb, + string_agg(b.lb_hab_fr,';' order by a.id_cb asc) lib_cb +FROM (SELECT distinct on (id_site,id_cb) * FROM zones_humides.r_site_habitat ) a JOIN ref_habitats.corine_biotope b ON a.id_cb = b.id - JOIN auteur c ON c.id_sitehab = a.id + JOIN d_max d on a.id_site = d.id_site WHERE a."valid" GROUP BY 1,2,3 -ORDER BY a.id_site, a."date" desc,row_number() OVER (ORDER BY a.id_site) desc ;""" grant = """ GRANT ALL ON TABLE zones_humides.v_zh_hab TO grp_admin; @@ -563,18 +586,35 @@ v_rhomeosite = """ DROP VIEW IF EXISTS zones_humides.v_rhomeosite; CREATE OR REPLACE VIEW zones_humides.v_rhomeosite AS -SELECT - v.site_code||' - '||v.nom "NAME", - SPLIT_PART(v.auteur_geom,' (',1) "REFERENT", - REPLACE(SPLIT_PART(v.auteur_geom,' (',2),')','') "ORG", - SPLIT_PART(v.typo_sdage,' - ',1) "TYPE", - CASE WHEN r.nom::text = 'alpin' THEN '1' - WHEN r.nom::text = 'continental' THEN '2' - WHEN r.nom::text = 'mediterraneen' THEN '4' - END "ODONATE", +with t1 as ( + SELECT + v.site_code, + r.nom, + st_area(st_intersection(v.geom, r.geom)) area_intersect + FROM zones_humides.v_zoneshumides v + join ref_territoire.ref_biogeo r on st_intersects(v.geom, r.geom) + --group by 1,2 +), t2 as ( + select + site_code, + nom, + row_number() over (partition by site_code order by area_intersect desc) as ismax_ter + from t1 +) +SELECT (v.site_code::text || ' - '::text) || v.nom::text AS "NAME", + split_part(v.auteur_geom, ' ('::text, 1) AS "REFERENT", + replace(split_part(v.auteur_geom, ' ('::text, 2), ')'::text, ''::text) AS "ORG", + split_part(v.typo_sdage, ' - '::text, 1) AS "TYPE", + CASE + WHEN t2.nom = 'alpin'::text THEN '1'::text + WHEN t2.nom = 'continental'::text THEN '2'::text + WHEN t2.nom = 'mediterraneen'::text THEN '4'::text + ELSE NULL::text + END AS "ODONATE", v.geom -FROM zones_humides.v_zoneshumides v, ref_territoire.ref_biogeo r -WHERE st_intersects(v.geom, r.geom) + FROM zones_humides.v_zoneshumides v + join t2 on v.site_code = t2.site_code and t2.ismax_ter = 1 + ; """ grant = """ GRANT ALL ON TABLE zones_humides.v_rhomeosite TO grp_admin; diff --git a/3_AZALEE/insert_ps.py b/3_AZALEE/insert_ps.py index fb9fb57..23e58ed 100644 --- a/3_AZALEE/insert_ps.py +++ b/3_AZALEE/insert_ps.py @@ -315,6 +315,7 @@ def normalize_colname(df): 'ident_':'ident', 'id':'ident', 'idfinal':'id_origine', + 'site_code':'id_site', 'date_':'date', 'obs':'auteur', 'structur':'structure', @@ -323,11 +324,13 @@ def normalize_colname(df): 's_p_brous' :'%_embrous', 's_p_brouss':'%_embrous', 'taux_embrou':'%_embrous', + 'tx_embrous':'%_embrous', 'niv__embro':'niv_embrous', 'niv_embro' :'niv_embrous', 'niv_embrou' :'niv_embrous', 'niv_emb' :'niv_embrous', 'embroussaillement' :'niv_embrous', + 'embrouss' :'niv_embrous', 'taux_recvmt':'%_recouvmnt', 'recouvrement':'recouvmnt', 'recouvreme':'recouvmnt', @@ -336,6 +339,7 @@ def normalize_colname(df): 'recouvr_' :'recouvmnt', 'remarque' :'remarques', 'remarq_' :'remarques', + 'rmq_intere' :'remarques', 'legendes' :'legende', 'legend' :'legende', 'sources' :'source', @@ -430,7 +434,7 @@ def ident_newsite(df,rcvmt=10): # Identification des superpositions new_site/old_site df_inters = gpd.sjoin(df,v_ps, how='left') del df_inters['index_right'] - news1 = df_inters[df_inters.site_code.isna()].id_origine + # news1 = df_inters[df_inters.site_code.isna()].id_origine lst_old_site = df_inters.site_code.unique() v_ps = v_ps[v_ps.site_code.isin(lst_old_site)].copy() v_ps.loc[:,'surf'] = v_ps.area @@ -1298,9 +1302,9 @@ if __name__ == "__main__": '"PS_VERCORS_CEN38_2011"' ] # from_table = '"cr_ECRIN_habitats_CBNA_2014"' - # from_table = '"cr_VERCORS_habitats_CBNA_1999-2007"' - from_file = 'PS_AGGREGATION_NB_AG.shp' - path0 = '/home/colas/Documents/9_PROJETS/2_PS/TO IMPORT/' + from_table = None + from_file = 'PS38_modifs_AG_2024.gpkg' + path0 = '/home/colas/Documents/9_PROJETS/2_PS/TO IMPORT/2024/' # org = from_file.split('/')[1] tutu = pd.DataFrame() @@ -1353,10 +1357,10 @@ if __name__ == "__main__": df = normalize_colname(df) df = format_date(df) df['table_org'] = table - df['structure'] = 'APIE' - df['type_pat'].replace(['Indéterminé','/'],None,inplace=True) - df.loc[df.type_pat.notna(),'pratiques'] = \ - df[df.type_pat.notna()].pratiques + ' ' + df[df.type_pat.notna()].type_pat.str.lower() + df['structure'] = 'CEN Isère' + # df['type_pat'].replace(['Indéterminé','/'],None,inplace=True) + # df.loc[df.type_pat.notna(),'pratiques'] = \ + # df[df.type_pat.notna()].pratiques + ' ' + df[df.type_pat.notna()].type_pat.str.lower() tutu = pd.concat([tutu,df]) @@ -1399,7 +1403,8 @@ if __name__ == "__main__": df_new = filter_saisierror(df_new) if not df_maj.empty: df_maj['auteur'] = normalize_auteur(df_maj.auteur) - df_maj['id_origine'] = df_maj['id_origine'].astype(str) + id_col = 'id_origine' if 'id_origine' in df_maj.columns else 'id_site' + df_maj['id_origine'] = df_maj[id_col].astype(str) df_maj = filter_saisierror(df_maj) if not df_replace['df'].empty: df_replace['df']['auteur'] = normalize_auteur(df_replace['df']['auteur']) @@ -1410,7 +1415,7 @@ if __name__ == "__main__": # # df[d].replace([' '], [' '],regex=True,inplace=True) # df[d] = df[d].str.normalize('NFKC') - DF = df_new.copy() + DF = df_maj.copy() Cnhab = DF.columns[DF.columns.str.startswith('n_hab')] if not all(DF[[*Cnhab]].astype(float).sum(axis=1) == 100): print( DF[ ~(DF[[*Cnhab]].astype(float).sum(axis=1) == 100) ] ) diff --git a/3_AZALEE/insert_zh.py b/3_AZALEE/insert_zh.py index 68c06d7..6ea517a 100644 --- a/3_AZALEE/insert_zh.py +++ b/3_AZALEE/insert_zh.py @@ -13,10 +13,6 @@ from pyproj import crs import pycen con = pycen.con -FILE_PATH = '/home/colas/Documents/9_PROJETS/1_ZH/MAJ/reinventairezhisre/' -GEOM_PATH = '20231011_Zone_Humide_Les_Chanines_Typhaie_Saint_Maurice_l_Exil.shp' -DATA_PATH = 'Tableau_saisie_ZH_Les_Chanines.xlsx' - def crsp_colSite(df): ''' @@ -756,6 +752,10 @@ def insertAttrsRegHydro(sh5): .replace([*p_inout_perm.nom.str.lower()],[*p_inout_perm.id.astype(str)]) if dfcon.columns.str.contains('sub').any(): + if dfcon['sub_freq'].dtype != object: + dfcon['sub_freq'] = dfcon['sub_freq'].astype(object) + if dfcon['sub_etend'].dtype != object: + dfcon['sub_etend'] = dfcon['sub_etend'].astype(object) dfcon['sub_freq'].fillna('Inconnu',inplace=True) dfcon['sub_etend'].fillna('Inconnu',inplace=True) dfcon['id_freqsub'] = dfcon['sub_freq'].str.lower() \ @@ -789,7 +789,7 @@ def insertAttrsRegHydro(sh5): if ins: ids = select_ID(dfcon[dfcon.columns.drop('auteur')],sch,tab_sub) ids.loc[~ids.id_etendsub.isna(),'id_etendsub'] = ids.loc[~ids.id_etendsub.isna(),'id_etendsub']\ - .astype(int).astype(str) + .astype(int) if ids.id_freqsub.dtype==int: ids.id_freqsub = ids.id_freqsub.astype(str) same_col = dfcon.columns[dfcon.columns.isin(ids.columns)] @@ -848,12 +848,13 @@ def insertAttrsFct(sh6,nom_typ_court=False): .str.lower() \ .replace([*param_tmp.nom.str.lower()],[*param_tmp.id]) df['id_fct'] = df['id_fct'].astype(int) + df,ins = insertAttrs(df,sch, tab) - if ins: ids = select_ID(df[df.columns.drop('auteur')],sch,tab) if 'description' in df.columns: df.description = df.description.astype(str) + df.loc[df.description=='nan','description'] = 'None' ids.description = ids.description.astype(str) if df.description.str.contains("'").any(): df.description = df.description.replace("'","''",regex=True) @@ -872,10 +873,16 @@ def insertAttrsFct(sh6,nom_typ_court=False): if __name__ == "__main__": - gdf = gpd.read_file(FILE_PATH+GEOM_PATH, crs='EPSG:2154') + from os import path + FILE_PATH = '/home/colas/Documents/9_PROJETS/1_ZH/MAJ/Actu 2024/TEREO - 20241002_ENVOI_SIG_ZH/ADD DATA' + GEOM_PATH = 'TEREO_newZH.gpkg' + DATA_PATH = 'Tableau_saisie_ZH_TEREO.xlsx' + + Gdf = gpd.read_file(path.join(FILE_PATH,GEOM_PATH), crs='EPSG:2154') + # Gdf['site_code'] = '38GR0070' # lst = ['38BB0089','38BB0090','38BB0091','38BB0092'] # gdf = gdf[gdf.site_code.isin(lst)] - DF = pd.read_excel(FILE_PATH+DATA_PATH, sheet_name=None, header=1) + DF = pd.read_excel(path.join(FILE_PATH,DATA_PATH), sheet_name=None, header=1) lst_sheet = [*DF.keys()] for k in lst_sheet: if isinstance(DF[k], pd.DataFrame): @@ -896,8 +903,9 @@ if __name__ == "__main__": inplace=True) lst_site = list(sh1.site_cod.unique()) - gdf = gdf[gdf.site_code.isin(lst_site)].reset_index(drop=True) - gdf.rename_geometry('geom', inplace=True) + gdf = Gdf[Gdf.site_code.isin(lst_site)].reset_index(drop=True) + if gdf.geometry.name != 'geom': + gdf.rename_geometry('geom', inplace=True) lst_site = list(gdf.site_code.unique()) sh1 = sh1[sh1.site_cod.isin(lst_site)] sh1.name = lst_sheet[0] @@ -979,7 +987,7 @@ if __name__ == "__main__": insertAttrsCB(sh2) insertAttrsUsgPrss(sh4) insertAttrsRegHydro(sh5) - insertAttrsFct(sh6) + insertAttrsFct(sh6.dropna(axis=1,how='all')) diff --git a/5_GEONATURE/EXPORT/flore_export_sinp_with_metadata.py b/5_GEONATURE/EXPORT/flore_export_sinp_with_metadata.py index 47aee33..075dc0c 100644 --- a/5_GEONATURE/EXPORT/flore_export_sinp_with_metadata.py +++ b/5_GEONATURE/EXPORT/flore_export_sinp_with_metadata.py @@ -1,188 +1,179 @@ from pycen import con_gn sql = ''' -DROP VIEW IF EXISTS gn_exports.v_synthese_sinp_with_metadata_flora; +DROP VIEW IF EXISTS gn_exports.v_synthese_sinp_with_metadata_flora_for_gn2pg; -CREATE VIEW gn_exports.v_synthese_sinp_with_metadata_flora AS -WITH af_actors AS ( - SELECT cafa.id_acquisition_framework, - json_build_object('type_role', - CASE - WHEN cafa.id_organism IS NOT NULL THEN 'organism'::TEXT - WHEN cafa.id_role IS NOT NULL THEN 'role'::TEXT - ELSE NULL::TEXT - END, - 'uuid_actor', coalesce(borg.uuid_organisme, tro.uuid_role), - 'cd_nomenclature_actor_role', tn.cd_nomenclature, 'identity', - CASE - WHEN cafa.id_organism IS NOT NULL - THEN json_build_object('organism_name', borg.nom_organisme) - WHEN cafa.id_role IS NOT NULL THEN json_build_object('first_name', - tro.nom_role, - 'last_name', - tro.prenom_role) END, - 'email', coalesce(borg.email_organisme, tro.email)) AS json_data - FROM gn_meta.cor_acquisition_framework_actor cafa - LEFT JOIN utilisateurs.bib_organismes borg ON cafa.id_organism = borg.id_organisme - LEFT JOIN utilisateurs.t_roles tro ON cafa.id_role = tro.id_role - JOIN ref_nomenclatures.t_nomenclatures tn - ON cafa.id_nomenclature_actor_role = tn.id_nomenclature), -af AS ( - SELECT taf.id_acquisition_framework, - jsonb_build_object('uuid', taf.unique_acquisition_framework_id, 'name', - taf.acquisition_framework_name, - 'desc', taf.acquisition_framework_desc, 'start_date', - taf.acquisition_framework_start_date, 'end_date', - taf.acquisition_framework_end_date, - 'initial_closing_date', taf.initial_closing_date, 'territorial_level', - ntl.cd_nomenclature, 'financing_type', nft.cd_nomenclature, 'target_description', - taf.target_description, 'ecologic_or_geologic_target', - taf.ecologic_or_geologic_target, 'actors', - json_agg(af_actors.json_data)) AS af_data - FROM gn_meta.t_acquisition_frameworks taf - JOIN af_actors ON af_actors.id_acquisition_framework = taf.id_acquisition_framework - LEFT JOIN ref_nomenclatures.t_nomenclatures ntl - ON taf.id_nomenclature_territorial_level = ntl.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures nft - ON taf.id_nomenclature_financing_type = nft.id_nomenclature - GROUP BY taf.id_acquisition_framework, taf.acquisition_framework_name, taf.acquisition_framework_desc, - taf.acquisition_framework_start_date, taf.acquisition_framework_end_date, taf.initial_closing_date, - ntl.cd_nomenclature, nft.cd_nomenclature), -ds_actors AS ( - SELECT cda.id_dataset, - json_build_object('type_role', - CASE - WHEN cda.id_organism IS NOT NULL THEN 'organism'::TEXT - WHEN cda.id_role IS NOT NULL THEN 'role'::TEXT - ELSE NULL::TEXT - END, 'uuid_actor', coalesce(borg.uuid_organisme, tro.uuid_role), - 'cd_nomenclature_actor_role', tn.cd_nomenclature, 'identity', - CASE - WHEN cda.id_organism IS NOT NULL - THEN json_build_object('organism_name', borg.nom_organisme) - WHEN cda.id_role IS NOT NULL THEN json_build_object('first_name', - tro.nom_role, - 'last_name', - tro.prenom_role) END, - 'email', coalesce(borg.email_organisme, tro.email)) AS json_data - FROM gn_meta.cor_dataset_actor cda - LEFT JOIN utilisateurs.bib_organismes borg ON cda.id_organism = borg.id_organisme - LEFT JOIN utilisateurs.t_roles tro ON cda.id_role = tro.id_role - JOIN ref_nomenclatures.t_nomenclatures tn - ON cda.id_nomenclature_actor_role = tn.id_nomenclature), - ds AS (SELECT tds.id_dataset, - tds.id_acquisition_framework, - -- tds.additional_data, - jsonb_build_object('uuid', tds.unique_dataset_id, 'name', tds.dataset_name, 'desc', tds.dataset_desc, - 'shortname', tds.dataset_shortname, 'data_type', ndt.cd_nomenclature, - 'collecting_method', ncm.cd_nomenclature, 'data_origin', ndo.cd_nomenclature, - 'dataset_objectif', ndso.cd_nomenclature, 'resource_type', nrt.cd_nomenclature, - 'source_status', nss.cd_nomenclature, 'territories', array_agg(DISTINCT - ref_nomenclatures.get_cd_nomenclature(cdt.id_nomenclature_territory)), - 'actors', json_agg(ds_actors.json_data)) AS dataset_data - FROM gn_meta.t_datasets tds - JOIN ds_actors ON ds_actors.id_dataset = tds.id_dataset - LEFT JOIN gn_meta.cor_dataset_territory cdt ON cdt.id_dataset = tds.id_dataset - LEFT JOIN ref_nomenclatures.t_nomenclatures ndt - ON tds.id_nomenclature_data_type = ndt.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures ncm - ON tds.id_nomenclature_collecting_method = ncm.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures ndo - ON tds.id_nomenclature_data_origin = ndo.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures ndso - ON tds.id_nomenclature_dataset_objectif = ndso.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures nrt - ON tds.id_nomenclature_resource_type = nrt.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures nss - ON tds.id_nomenclature_source_status = nss.id_nomenclature - GROUP BY tds.id_dataset, tds.id_acquisition_framework, tds.unique_dataset_id, tds.dataset_name, - tds.dataset_desc, tds.dataset_shortname, ndt.cd_nomenclature, ncm.cd_nomenclature, - ndo.cd_nomenclature, ndso.cd_nomenclature, nrt.cd_nomenclature, nss.cd_nomenclature) -SELECT s.id_synthese, - s.entity_source_pk_value AS id_source, - s.unique_id_sinp AS id_perm_sinp, - s.unique_id_sinp_grp AS id_perm_grp_sinp, - s.date_min AS date_debut, - s.date_max AS date_fin, - s.cd_nom, - s.meta_v_taxref AS version_taxref, - s.nom_cite, - s.count_min AS nombre_min, - s.count_max AS nombre_max, - s.altitude_min, - s.altitude_max, - s.depth_min AS profondeur_min, - s.depth_max AS profondeur_max, - s.observers AS observateurs, - s.determiner AS determinateur, - s.validator AS validateur, - s.sample_number_proof AS numero_preuve, - s.digital_proof AS preuve_numerique, - s.non_digital_proof AS preuve_non_numerique, - s.comment_context AS comment_releve, - s.comment_description AS comment_occurrence, - ds.dataset_data AS jdd_data, - af.af_data AS ca_data, - s.reference_biblio, - s.cd_hab AS code_habitat, - h.lb_hab_fr AS habitat, - s.place_name AS nom_lieu, - s.precision, - s.additional_data::TEXT AS donnees_additionnelles, - st_astext(s.the_geom_4326) AS wkt_4326, - n1.cd_nomenclature AS nature_objet_geo, - n2.cd_nomenclature AS type_regroupement, - s.grp_method AS methode_regroupement, - n3.cd_nomenclature AS comportement, - n4.cd_nomenclature AS technique_obs, - n5.cd_nomenclature AS statut_biologique, - n6.cd_nomenclature AS etat_biologique, - n7.cd_nomenclature AS naturalite, - n8.cd_nomenclature AS preuve_existante, - n9.cd_nomenclature AS precision_diffusion, - n10.cd_nomenclature AS stade_vie, - n11.cd_nomenclature AS sexe, - n12.cd_nomenclature AS objet_denombrement, - n13.cd_nomenclature AS type_denombrement, - n14.cd_nomenclature AS niveau_sensibilite, - n15.cd_nomenclature AS statut_observation, - n16.cd_nomenclature AS floutage_dee, - n17.cd_nomenclature AS statut_source, - n18.cd_nomenclature AS type_info_geo, - n19.cd_nomenclature AS methode_determination, - n20.cd_nomenclature AS statut_validation, - coalesce(s.meta_update_date, s.meta_create_date) AS derniere_action -FROM gn_synthese.synthese s - JOIN ds ON ds.id_dataset = s.id_dataset - JOIN af ON ds.id_acquisition_framework = af.id_acquisition_framework - LEFT JOIN ref_habitats.habref h ON h.cd_hab = s.cd_hab - LEFT JOIN ref_nomenclatures.t_nomenclatures n1 ON s.id_nomenclature_geo_object_nature = n1.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures n2 ON s.id_nomenclature_grp_typ = n2.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures n3 ON s.id_nomenclature_behaviour = n3.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures n4 ON s.id_nomenclature_obs_technique = n4.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures n5 ON s.id_nomenclature_bio_status = n5.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures n6 ON s.id_nomenclature_bio_condition = n6.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures n7 ON s.id_nomenclature_naturalness = n7.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures n8 ON s.id_nomenclature_exist_proof = n8.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures n9 ON s.id_nomenclature_diffusion_level = n9.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures n10 ON s.id_nomenclature_life_stage = n10.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures n11 ON s.id_nomenclature_sex = n11.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures n12 ON s.id_nomenclature_obj_count = n12.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures n13 ON s.id_nomenclature_type_count = n13.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures n14 ON s.id_nomenclature_sensitivity = n14.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures n15 ON s.id_nomenclature_observation_status = n15.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures n16 ON s.id_nomenclature_blurring = n16.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures n17 ON s.id_nomenclature_source_status = n17.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures n18 ON s.id_nomenclature_info_geo_type = n18.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures n19 ON s.id_nomenclature_determination_method = n19.id_nomenclature - LEFT JOIN ref_nomenclatures.t_nomenclatures n20 ON s.id_nomenclature_valid_status = n20.id_nomenclature +CREATE VIEW gn_exports.v_synthese_sinp_with_metadata_flora_for_gn2pg AS + WITH af_actors AS ( + SELECT cafa.id_acquisition_framework + FROM gn_meta.cor_acquisition_framework_actor cafa + LEFT JOIN utilisateurs.bib_organismes borg ON cafa.id_organism = borg.id_organisme + WHERE borg.uuid_organisme = '5a433bd0-2070-25d9-e053-2614a8c026f8'::uuid + ), af AS ( + SELECT taf.id_acquisition_framework + FROM gn_meta.t_acquisition_frameworks taf + JOIN af_actors ON af_actors.id_acquisition_framework = taf.id_acquisition_framework + GROUP BY taf.id_acquisition_framework + ), ds AS ( + SELECT tds.id_dataset, + tds.dataset_name AS nom_jdd, + tds.id_acquisition_framework + FROM gn_meta.t_datasets tds + GROUP BY tds.id_dataset, tds.dataset_name + ), geo AS ( + SELECT "left"(geo_1.area_code::text, 2) AS departement, + geo_1.area_code AS commune, + s_1.id_synthese, + st_transform(s_1.the_geom_local, 4326) AS st_transform + FROM ref_geo.l_areas geo_1 + JOIN gn_synthese.synthese s_1 ON st_intersects(s_1.the_geom_4326, st_transform(geo_1.geom, 4326)) + WHERE geo_1.id_type = 25 + ) + SELECT + ds.nom_jdd AS nom_jdd, + s.unique_id_sinp_grp AS id_sinp_releve, + occ.id_releve_occtax AS identifiant_releve, + NULL::text AS code_perso_releve, + s.unique_id_sinp AS id_sinp_observation, + s.entity_source_pk_value AS identifiant_observation, + geo.departement, + geo.commune, + NULL::text AS lieu_dit, + CASE + WHEN "position"(sp.srtext::text, 'GEOGCS'::text) = 1 THEN "substring"(replace(sp.srtext::text, 'GEOGCS["'::text, ''::text), 1, "position"(replace(sp.srtext::text, 'GEOGCS["'::text, ''::text), '",'::text) - 1) + WHEN "position"(sp.srtext::text, 'PROJCS'::text) = 1 THEN "substring"(replace(sp.srtext::text, 'PROJCS["'::text, ''::text), 1, "position"(replace(sp.srtext::text, 'PROJCS["'::text, ''::text), '",'::text) - 1) + WHEN "position"(sp.srtext::text, 'GEOCCS'::text) = 1 THEN "substring"(replace(sp.srtext::text, 'GEOCCS["'::text, ''::text), 1, "position"(replace(sp.srtext::text, 'GEOCCS["'::text, ''::text), '",'::text) - 1) + WHEN "position"(sp.srtext::text, 'COMPD_CS'::text) = 1 THEN "substring"(replace(sp.srtext::text, 'COMPD_CS["'::text, ''::text), 1, "position"(replace(sp.srtext::text, 'COMPD_CS["'::text, ''::text), '",'::text) - 1) + ELSE 'Non défini'::text + END AS sys_coord, + st_srid(s.the_geom_4326) AS srid, + st_astext(s.the_geom_4326) AS localisation_wkt, + st_x(st_centroid(s.the_geom_4326)) AS coord_x, + st_y(st_centroid(s.the_geom_4326)) AS coord_y, + s."precision" AS precision, + NULL::text AS nature_objet, + s.altitude_min AS alti_min, + s.altitude_max AS alti_max, + NULL::text AS pente, + NULL::text AS exposition, + NULL::text AS comm_geol, + NULL::text AS milieu, + s.observers AS observateurs, + s.date_min AS date_debut, + s.date_max AS date_fin, + NULL::text AS comm_context, + n2.mnemonique AS type_regroupement, + s.grp_method AS meth_regroupement, + NULL::text AS surface, + CASE WHEN (s."additional_data"::json#>>'{strate_flore}') IS NOT NULL + THEN (s."additional_data"::json#>>'{strate_flore}')::text + WHEN s."additional_data"::json#>>'{id_nomenclature_strate_flore}' IS NOT NULL + THEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text + ELSE NULL::text END AS strate_vegetation, + CASE WHEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text = 'Strate herbacée'::text + THEN (s."additional_data"::json#>>'{heigth_herbace}')::text + WHEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text = 'Strate arbustive'::text + THEN (s."additional_data"::json#>>'{heigth_arbust}')::text + WHEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text = 'Strate arborée'::text + THEN (s."additional_data"::json#>>'{heigth_arbore}')::text + WHEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text = 'Strate sous-arbustive'::text + THEN (s."additional_data"::json#>>'{heigth_ssarbust}')::text + ELSE NULL::text END + AS hauteur_strate, + CASE WHEN s."additional_data"::json#>>'{id_nomenclature_strate_flore}' IS NOT NULL + THEN CASE WHEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text = 'Strate herbacée'::text + THEN (s."additional_data"::json#>>'{rcvmt_herbace}')::text + WHEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text = 'Strate arbustive'::text + THEN (s."additional_data"::json#>>'{rcvmt_arbust}')::text + WHEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text = 'Strate arborée'::text + THEN (s."additional_data"::json#>>'{rcvmt_arbore}')::text + WHEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text = 'Strate sous-arbustive'::text + THEN (s."additional_data"::json#>>'{rcvmt_ssarbust}')::text + ELSE NULL::text END + ELSE NULL::text END AS recouvrement_strate, + s.cd_hab AS cdhab, + NULL::text AS cdhab_v, + NULL::text AS code_eur, + NULL::text AS code_eunis, + NULL::text AS code_cahab, + NULL::text AS code_cb, + NULL::text AS id_microhab, + n21.regne AS regne, + s.nom_cite AS nom_cite, + s.cd_nom as cd_nom, + CASE WHEN (s."additional_data"::json#>>'{effectif_textuel}') IS NOT NULL + THEN (s."additional_data"::json#>>'{effectif_textuel}')::text + WHEN s."additional_data"::json#>>'{id_nomenclature_braunblanq_abdom}' IS NOT NULL + THEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_braunblanq_abdom}')::int)::text + ELSE NULL::text END AS abondance, + NULL::text AS sociabilite, + n11.mnemonique AS sexe, + n7.mnemonique AS naturalite, + NULL::text AS comm_description, + n15.mnemonique AS statut_observation, + n12.mnemonique AS objet_denombrement, + n13.mnemonique AS type_denombrement, + s.count_min AS nombre_min, + s.count_max AS nombre_max, + n17.label_default AS statut_source, + NULL::text AS reference_biblio, + NULL::text AS page, + n8.mnemonique AS preuve_existence, + s.digital_proof AS preuve_numerique, + s.non_digital_proof AS preuve_non_numerique, + NULL::text AS nom_collection, + NULL::text AS ref_collection, + s.determiner AS determinateur, + NULL::text AS niv_val, + NULL::text AS niveau_diffusion, + n16.mnemonique AS floutage_dee, + NULL::text AS methode_observation, + CASE WHEN ts.name_source like 'MONITORING_%%' + THEN split_part(ts.name_source,'_',2) + ELSE 'RELEVE FLORE'::text END AS protocole, + n6.mnemonique AS etat_biologique, + n5.mnemonique AS statut_biologique, + n10.mnemonique AS stade_vie, + n19.mnemonique AS methode_determination, + n3.mnemonique AS comportement, + s.additional_data - '{effectif_textuel,strate_flore,code_atlas,id_nomenclature_braunblanq_abdom,id_nomenclature_strate_flore}'::text[] AS additional_data + FROM gn_synthese.synthese s + JOIN ds ON ds.id_dataset = s.id_dataset + JOIN af ON ds.id_acquisition_framework = af.id_acquisition_framework + JOIN geo ON s.id_synthese = geo.id_synthese + JOIN spatial_ref_sys sp ON st_srid(s.the_geom_4326) = sp.auth_srid + LEFT JOIN ref_habitats.habref h ON h.cd_hab = s.cd_hab + LEFT JOIN pr_occtax.t_releves_occtax occ ON occ.unique_id_sinp_grp = s.unique_id_sinp_grp + LEFT JOIN ref_nomenclatures.t_nomenclatures n1 ON s.id_nomenclature_geo_object_nature = n1.id_nomenclature + LEFT JOIN ref_nomenclatures.t_nomenclatures n2 ON s.id_nomenclature_grp_typ = n2.id_nomenclature + LEFT JOIN ref_nomenclatures.t_nomenclatures n3 ON s.id_nomenclature_behaviour = n3.id_nomenclature + LEFT JOIN ref_nomenclatures.t_nomenclatures n4 ON s.id_nomenclature_obs_technique = n4.id_nomenclature + LEFT JOIN ref_nomenclatures.t_nomenclatures n5 ON s.id_nomenclature_bio_status = n5.id_nomenclature + LEFT JOIN ref_nomenclatures.t_nomenclatures n6 ON s.id_nomenclature_bio_condition = n6.id_nomenclature + LEFT JOIN ref_nomenclatures.t_nomenclatures n7 ON s.id_nomenclature_naturalness = n7.id_nomenclature + LEFT JOIN ref_nomenclatures.t_nomenclatures n8 ON s.id_nomenclature_exist_proof = n8.id_nomenclature + LEFT JOIN ref_nomenclatures.t_nomenclatures n9 ON s.id_nomenclature_diffusion_level = n9.id_nomenclature + LEFT JOIN ref_nomenclatures.t_nomenclatures n10 ON s.id_nomenclature_life_stage = n10.id_nomenclature + LEFT JOIN ref_nomenclatures.t_nomenclatures n11 ON s.id_nomenclature_sex = n11.id_nomenclature + LEFT JOIN ref_nomenclatures.t_nomenclatures n12 ON s.id_nomenclature_obj_count = n12.id_nomenclature + LEFT JOIN ref_nomenclatures.t_nomenclatures n13 ON s.id_nomenclature_type_count = n13.id_nomenclature + LEFT JOIN ref_nomenclatures.t_nomenclatures n14 ON s.id_nomenclature_sensitivity = n14.id_nomenclature + LEFT JOIN ref_nomenclatures.t_nomenclatures n15 ON s.id_nomenclature_observation_status = n15.id_nomenclature + LEFT JOIN ref_nomenclatures.t_nomenclatures n16 ON s.id_nomenclature_blurring = n16.id_nomenclature + LEFT JOIN ref_nomenclatures.t_nomenclatures n17 ON s.id_nomenclature_source_status = n17.id_nomenclature + LEFT JOIN ref_nomenclatures.t_nomenclatures n18 ON s.id_nomenclature_info_geo_type = n18.id_nomenclature + LEFT JOIN ref_nomenclatures.t_nomenclatures n19 ON s.id_nomenclature_determination_method = n19.id_nomenclature + LEFT JOIN ref_nomenclatures.t_nomenclatures n20 ON s.id_nomenclature_valid_status = n20.id_nomenclature + LEFT JOIN taxonomie.taxref n21 ON s.cd_nom = n21.cd_nom + JOIN gn_synthese.cor_area_synthese cas ON cas.id_synthese = s.id_synthese - JOIN taxonomie.taxref t ON t.cd_nom = s.cd_nom + -- ADD CEN_38 JOIN gn_synthese.t_sources ts ON ts.id_source = s.id_source - -WHERE t.regne <> 'Animalia' + WHERE n21.regne::text = 'Plantae'::text -- exclision CA ([SICEN] Données anciennes,[TEST] : migration sicen,[SICEN] Hors Etude) - AND ds.id_acquisition_framework NOT IN (53,66,1,2,6) + AND ds.id_acquisition_framework NOT IN (53,65,66,1,2,6) + -- exclision JDD `Observations opportunistes du CEN Isère importé depuis Faune Isère` + AND ds.id_dataset NOT IN (185,377,236) -- exclision CA (`Gestion de l'Espace Naturel Sensible du Méandre des Oves`, `Gestion de la Réserve Naturelle Nationale de l'Ile de la Platière`, `Gestion des Natura 2000 FR 8201749 « Milieux alluviaux et aquatiques de l’Île de la Platière » et FR 8212012 « Île de la Platière »`, `RNN Platière` --AND (ts.id_module = 7 AND ds.id_acquisition_framework IN (7,38,39,42,44,45,46,48,55,57,58,60) AND ( @@ -191,12 +182,10 @@ WHERE t.regne <> 'Animalia' ) AND unaccent(observers) NOT LIKE ALL(array[ '%%Sympetrum%%','%%Departement%%','%%Rhone%%','%%Oxalis%%', - '%%LPO%%','%%GRPLS%%','%%Parvi%%','%%CD Isere%%','%%CBNA%%', - '%%Flavia%%','%%Gentiana%%','%%region%%','%%Personnel%%','%%Independant%%' + '%%LPO%%','%%GRPLS%%','%%Parvi%%','%%CD Isere%%','%%Personnel%%','%%Independant%%' + '%%DELOCHE Denis%%' ]) AND observers NOT IN ('', 'Benoit Dodelin',', ') - -- exclision JDD `Observations opportunistes du CEN Isère importé depuis Faune Isère` - AND ds.id_dataset NOT IN (185,377) -- statut_validation = 'Certain - très probable' -- AND n20.cd_nomenclature = '1' ORDER BY s.id_synthese; diff --git a/5_GEONATURE/insert_liste_alerte.py b/5_GEONATURE/insert_liste_alerte.py index aee0413..0b82c0d 100644 --- a/5_GEONATURE/insert_liste_alerte.py +++ b/5_GEONATURE/insert_liste_alerte.py @@ -23,6 +23,11 @@ def test_status_type(con,col,status): return cnx.execute(sql).one()[0] def insert_status_alerte(con): + """ + Insert type de statut pour les alertes + + Si la valeur AL n'existe pas dans la table bdc_statut_type, on l'insert + """ if test_data(con,'bdc_statut_type','cd_type_statut','AL') > 0: # if test_status_type(con,'cd_type_statut','AL') > 0: print('ALERTE STATUS ALREADY EXISTS') @@ -35,6 +40,23 @@ def insert_status_alerte(con): cnx.execute(sql) def insert_status_values(con): + """ + Inserts predefined status values into the 'bdc_statut_values' table if they do not already exist. + + This function iterates over a list of status values, checking whether each value already exists in the + specified database table. If a value does not exist, it inserts the value into the table. Status values + include a code and a label, which describe the extinction risk or conservation status of a taxonomic + group at the departmental level. + + Args: + con: A SQLAlchemy connection object to the database. + + Note: + This function assumes the existence of a schema named 'taxonomie' and a table named + 'bdc_statut_values' in the database connected via 'con'. It also assumes the availability of a + 'con_gn' connection for executing the SQL commands. + """ + vals = [ ['RE','Disparue au niveau départemental'], ['AS-1','Quasi menacée (localisées sans signe de déclin)'], @@ -133,15 +155,22 @@ def test_status_text(con,col,cd_doc): with con.begin() as cnx: return cnx.execute(sql).one()[0] -def insert_statut_text(con,cd_doc,doc): +def insert_statut_text(con,cd_doc,doc,cd_sig,lb_adm_tr,niveau_admin): if test_data(con,'bdc_statut_text','cd_doc',cd_doc['id_doc']) > 0: - # if test_status_text(con,'cd_doc',cd_doc) > 0: print('ALERTE TEXT STATUS ALREADY EXISTS : ',doc) else: sql = ''' - INSERT INTO {sch}.{tab} (cd_type_statut,cd_sig,cd_doc,niveau_admin,lb_adm_tr,doc_url,enable) VALUES - ('AL','{cd_doc}','INSEED38','Département','Isère','{doc}',TRUE) - ;'''.format(sch='taxonomie',tab='bdc_statut_text',cd_doc=cd_doc['id_doc'],doc=doc) + INSERT INTO {sch}.{tab} (cd_type_statut,cd_doc,cd_sig,niveau_admin,lb_adm_tr,doc_url,enable) VALUES + ('AL',{cd_doc},'{cdsig}','{lb_adm}','{niv_adm}','{doc}',TRUE) + ;'''.format( + sch='taxonomie', + tab='bdc_statut_text', + cd_doc=cd_doc['id_doc'], + doc=doc, + cdsig=cd_sig, + lb_adm=lb_adm_tr, + niv_adm=niveau_admin + ) with con_gn.begin() as cnx: cnx.execute(sql) @@ -162,7 +191,6 @@ def get_max_idstatuttaxo(con): with con.begin() as cnx: return cnx.execute(sql).one()[0] - def insert_status_taxo(con,cd_nom,cd_doc,status): id_statut_cor = get_id_status_cor_text_values(con,cd_doc,status)[0][0] cd_ref = get_cd_ref(con,cd_nom) @@ -199,16 +227,50 @@ def get_taxonomie(con,cd_nom): ;'''.format(sch='taxonomie',tab='taxref',cd_nom=tuple(cd_nom)).replace(",)",")") return pd.read_sql(sql,con) + if __name__ == "__main__": - from pycen import con_gn, ref_hydro + import pandas as pd + # Définition de la connection à la bdd GéoNature + from pycen import con_gn + + # Le premier feuillet du fichier Excel lu. + # Composition minimale: [CD_NOM,Statut, Source, Source_url] + # ATTENTION: + # - le CD_NOM doit correspondre au CD_NOM de la table taxref + # - Les taxons dont la Source_url est None ou NA seront ignorés file = '/home/colas/Documents/9_PROJETS/6_GEONATURE/listes_alertes_isère.xlsx' + # Echelle administratif des listes à implémenter + niveau_admin = 'Département' + # Nom du niveau adminsitratif + lb_adm_tr = 'Isère' + # Code SIG du niveau adminnistratif + cd_sig = 'INSEED38' insert_status_alerte(con_gn) + # Dictionnaire des listes d’alerte à intégrer, + # identifiant et codes status respectivement concernés cd_doc = { - 'Statut_de_conservation_des_poissons_et_écrevisses_en_Isère_2015':{'id_doc':999990,'id_values':['RE','CR','EN','VU','NT','LC','DD','NA',]}, - 'Liste_d’alerte_sur_les_orthoptères_menacés_en_Isère_2014':{'id_doc':999991,'id_values':['RE','CR','EN','VU','AS-1','AS-2','AS-3','LC','DD','NA']}, - 'Statuts_de_conservation_de_la_faune_sauvage_en_isere_2016':{'id_doc':999992,'id_values':['RE','CR','EN','VU','NT','LC','DD','NA','NE',]}, + 'Statut_de_conservation_des_poissons_et_écrevisses_en_Isère_2015':{ + 'id_doc':999990, + 'id_values':['RE','CR','EN','VU','NT','LC','DD','NA',] + }, + 'Liste_d’alerte_sur_les_orthoptères_menacés_en_Isère_2014':{ + 'id_doc':999991, + 'id_values':['RE','CR','EN','VU','AS-1','AS-2','AS-3','LC','DD','NA'] + }, + 'Statuts_de_conservation_de_la_faune_sauvage_en_isere_2016':{ + 'id_doc':999992, + 'id_values':['RE','CR','EN','VU','NT','LC','DD','NA','NE',] + }, + 'Liste_rouge_des_Odonates_de_l’Isère_2013':{ + 'id_doc':999993, + 'id_values':['RE','CR','EN','VU','NT','LC','DD','NA',] + }, + 'Liste_rouge_des_lépidoprere_rhopaloceres_et_zygenes_de_l’Isère_2015':{ + 'id_doc':999994, + 'id_values':['RE','CR','EN','VU','NT','LC','DD','NA','NE','EX'] + }, } df = (pd.read_excel(file,keep_default_na=False) @@ -221,12 +283,18 @@ if __name__ == "__main__": df = df.loc[df.doc_url!=''] for d in df.source.unique(): doc_url = df.loc[df.source==d,'doc_url'].unique()[0] - insert_statut_text(con_gn,cd_doc[d],doc_url) + insert_statut_text(con_gn,cd_doc[d],doc_url,cd_sig,lb_adm_tr,niveau_admin) # INSERTION dans la table bdc_statut_taxons - for row in df.itertuples(): - id_doc = cd_doc[row.source]['id_doc'] - insert_status_taxo(con=con_gn,cd_nom=row.cd_nom,cd_doc=id_doc,status=row.code_statut) + # Boucle sur chaque taxon. Peu prendre quelques minutes. + [ + insert_status_taxo( + con = con_gn, + cd_nom = row.cd_nom, + cd_doc = cd_doc[row.source]['id_doc'], + status = row.code_statut) + for row in df.itertuples() + ] st = get_status_type(con_gn,'cd_type_statut','AL') for c in st: @@ -237,7 +305,7 @@ if __name__ == "__main__": del df['nom_français'] del df['nom_latin'] df = df.merge(tax,how='inner',on='cd_nom') - df['cd_sig'] = 'INSEED38' - df['lb_adm_tr'] = 'Isère' - df['niveau_admin'] = 'Département' + df['cd_sig'] = cd_sig + df['lb_adm_tr'] = lb_adm_tr + df['niveau_admin'] = niveau_admin df.to_sql('bdc_statut',con_gn,schema='taxonomie',if_exists='append',index=False) diff --git a/5_GEONATURE/pivot_bdc_status_v2.py b/5_GEONATURE/pivot_bdc_status_v2.py index 6a5fa97..3a31033 100644 --- a/5_GEONATURE/pivot_bdc_status_v2.py +++ b/5_GEONATURE/pivot_bdc_status_v2.py @@ -1,10 +1,12 @@ -from pycen import con_gn +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- + import requests import numpy as np import pandas as pd import os -def get_status(lst): +def get_status(lst,con): sql = """ SELECT t.cd_nom, @@ -16,6 +18,7 @@ def get_status(lst): t.famille, t.group1_inpn, t.group2_inpn, + t.group3_inpn, t.nom_vern, t.nom_complet, t.nom_valide, @@ -25,25 +28,32 @@ def get_status(lst): s.code_statut, s.cd_type_statut, s.label_statut, + s.niveau_admin, s.full_citation, s.doc_url FROM taxonomie.taxref t JOIN taxonomie.v_bdc_status s USING (cd_nom) WHERE t.cd_nom IN {cd_nom} ;""".format(cd_nom = tuple(lst)) - return pd.read_sql_query(sql,con_gn) + return pd.read_sql_query(sql,con) -def get_api_status(cd_nom:int): - res = requests.api.get('https://geonature.cen-isere.fr/taxhub/api/taxref/%i'%cd_nom) +def get_type_status(con): + sql = """ + SELECT * FROM taxonomie.bdc_statut_type + ;""" + return pd.read_sql_query(sql,con) + +def get_api_status(api,cd_nom:int): + res = requests.api.get('%s/%i'%(api,cd_nom)) if res.status_code == 200: return res.json() else : raise('Error : %i\tcd_nom : %i'%(res.status_code,cd_nom)) -def get_taxon_status(lst): +def get_taxon_status(lst,api): from datetime import datetime as dt init = dt.now() - st = [get_api_status(x) for x in lst] # TOO LONG + st = [get_api_status(api,x) for x in lst] # TOO LONG print(dt.now()-init) phylo = { 'cd_ref':[x['cd_ref'] for x in st], @@ -123,12 +133,6 @@ def get_taxon_status(lst): if 'PD' in x['status'].keys() else None for x in st ], - 'PD':[ - [val['values'][v]['code_statut'] - for val in x['status']['PD']['text'].values() for v in val['values'] ] - if 'PD' in x['status'].keys() else None - for x in st - ], 'PNA':[ [val['values'][v]['code_statut'] for val in x['status']['PNA']['text'].values() for v in val['values'] ] @@ -194,149 +198,7 @@ def get_taxon_status(lst): for val in x['status']['exPNA']['text'].values() for v in val['values'] ] if 'exPNA' in x['status'].keys() else None for x in st - ], - - } - cd_status = { - 'AL':[ - [val['values'][v]['code_statut'] - for val in x['status']['AL']['text'].values() for v in val['values'] ] - if 'AL' in x['status'].keys() else None - for x in st - ], - 'BERN':[ - [val['values'][v]['code_statut'] - for val in x['status']['BERN']['text'].values() for v in val['values'] ] - if 'BERN' in x['status'].keys() else None - for x in st - ], - 'BONN':[ - [val['values'][v]['code_statut'] - for val in x['status']['BONN']['text'].values() for v in val['values'] ] - if 'BONN' in x['status'].keys() else None - for x in st - ], - 'DH':[ - [val['values'][v]['code_statut'] - for val in x['status']['DH']['text'].values() for v in val['values'] ] - if 'DH' in x['status'].keys() else None - for x in st - ], - 'DO':[ - [val['values'][v]['code_statut'] - for val in x['status']['DO']['text'].values() for v in val['values'] ] - if 'DO' in x['status'].keys() else None - for x in st - ], - 'LRE':[ - [val['values'][v]['code_statut'] - for val in x['status']['LRE']['text'].values() for v in val['values'] ] - if 'LRE' in x['status'].keys() else None - for x in st - ], - 'LRM':[ - [val['values'][v]['code_statut'] - for val in x['status']['LRM']['text'].values() for v in val['values'] ] - if 'LRM' in x['status'].keys() else None - for x in st - ], - 'LRN':[ - [val['values'][v]['code_statut'] - for val in x['status']['LRN']['text'].values() for v in val['values'] ] - if 'LRN' in x['status'].keys() else None - for x in st - ], - 'LRR':[ - [val['values'][v]['code_statut'] - for val in x['status']['LRR']['text'].values() for v in val['values'] ] - if 'LRR' in x['status'].keys() else None - for x in st - ], - 'PAPNAT':[ - [val['values'][v]['code_statut'] - for val in x['status']['PAPNAT']['text'].values() for v in val['values'] ] - if 'PAPNAT' in x['status'].keys() else None - for x in st - ], - 'PD':[ - [val['values'][v]['code_statut'] - for val in x['status']['PD']['text'].values() for v in val['values'] ] - if 'PD' in x['status'].keys() else None - for x in st - ], - 'PD':[ - [val['values'][v]['code_statut'] - for val in x['status']['PD']['text'].values() for v in val['values'] ] - if 'PD' in x['status'].keys() else None - for x in st - ], - 'PNA':[ - [val['values'][v]['code_statut'] - for val in x['status']['PNA']['text'].values() for v in val['values'] ] - if 'PNA' in x['status'].keys() else None - for x in st - ], - 'PR':[ - [val['values'][v]['code_statut'] - for val in x['status']['PR']['text'].values() for v in val['values'] ] - if 'PR' in x['status'].keys() else None - for x in st - ], - 'REGL':[ - [val['values'][v]['code_statut'] - for val in x['status']['REGL']['text'].values() for v in val['values'] ] - if 'REGL' in x['status'].keys() else None - for x in st - ], - 'REGLII':[ - [val['values'][v]['code_statut'] - for val in x['status']['REGLII']['text'].values() for v in val['values'] ] - if 'REGLII' in x['status'].keys() else None - for x in st - ], - 'REGLLUTTE':[ - [val['values'][v]['code_statut'] - for val in x['status']['REGLLUTTE']['text'].values() for v in val['values'] ] - if 'REGLLUTTE' in x['status'].keys() else None - for x in st - ], - 'REGLSO':[ - [val['values'][v]['code_statut'] - for val in x['status']['REGLSO']['text'].values() for v in val['values'] ] - if 'REGLSO' in x['status'].keys() else None - for x in st - ], - 'SCAP NAT':[ - [val['values'][v]['code_statut'] - for val in x['status']['SCAP NAT']['text'].values() for v in val['values'] ] - if 'SCAP NAT' in x['status'].keys() else None - for x in st - ], - 'SCAP REG':[ - [val['values'][v]['code_statut'] - for val in x['status']['SCAP REG']['text'].values() for v in val['values'] ] - if 'SCAP REG' in x['status'].keys() else None - for x in st - ], - 'SENSNAT':[ - [val['values'][v]['code_statut'] - for val in x['status']['SENSNAT']['text'].values() for v in val['values'] ] - if 'SENSNAT' in x['status'].keys() else None - for x in st - ], - 'ZDET':[ - [val['values'][v]['code_statut'] - for val in x['status']['ZDET']['text'].values() for v in val['values'] ] - if 'ZDET' in x['status'].keys() else None - for x in st - ], - 'exPNA':[ - [val['values'][v]['code_statut'] - for val in x['status']['exPNA']['text'].values() for v in val['values'] ] - if 'exPNA' in x['status'].keys() else None - for x in st - ], - + ] } return pd.DataFrame({**phylo,**cd_status}) @@ -349,26 +211,40 @@ dict_dep = { if __name__ == "__main__": + # Définition de la connection à la bdd GéoNature + from pycen import con_gn + # NOT USE FOR NOW - API Taxref + api_taxref = 'https://geonature.cen-isere.fr/taxhub/api/taxref' - PATH = '/media/colas/SRV/FICHIERS/SITES/SITES GERES/ROLA_ROLANDE-MAUPAS/ROLA_PPI/ROLA_2025-2034_PG/donneesnaturalistes' - file = '3_liste sp_ROLA.xlsx' - sheet = 'liste sp' + # Paramètres de chargement du fichier des taxons + PATH = '/home/colas/Documents/tmp/CHARVAS' + file = 'liste_sp_CHAR.xlsx' + sheet = 'liste_sp' # Liste des CD_NOM en entrée - cd_col = 'cd_ref' + cd_col = 'cd_ref' # Nom de la colonne à utiliser dans le feuillet ``sheet`` + + # Lecture des données taxlist = pd.read_excel(os.path.join(PATH,file),sheet,usecols=[cd_col],header=0) tab_sp = pd.read_excel(os.path.join(PATH,file),sheet,index_col=cd_col) lst = taxlist[cd_col] - df = get_status(taxlist[cd_col].astype(str)) + # Récupération des statuts + df = get_status(taxlist[cd_col].astype(str),con_gn) + typ = get_type_status(con_gn) + typ = typ[typ.cd_type_statut.isin(df.cd_type_statut.unique())] + + # Distinction LRR [old vs new] région + is_lrr = df.cd_type_statut == 'LRR' + df.loc[is_lrr & (df.niveau_admin == 'Région'),'cd_type_statut'] = 'LRR_AURA' + df.loc[is_lrr & (df.niveau_admin == 'Ancienne région'),'cd_type_statut'] = 'LRR_RA' + del df['niveau_admin'] for c in ['cd_ref','cd_nom','lb_nom']: if c in tab_sp.columns: # if 'cd_nom' not in df.columns and c == 'cd_ref': continue tab_sp.drop(c,axis=1,inplace=True) - # df.to_csv('/media/colas/SRV/FICHIERS/TRANSFERTS-EQUIPE/LC/BOCA_CD_NOM_STATUS.csv') - pivot = pd.pivot_table( df, values='code_statut', @@ -385,11 +261,11 @@ if __name__ == "__main__": pivot = tab_sp.merge(pivot,on=[cd_col],how='left') - pivlib = pd.pivot_table( df, values='label_statut', - index=['cd_nom', 'cd_ref','lb_nom'#,'niveau_admin','lb_adm_tr' + index=[ + 'cd_nom', 'cd_ref','lb_nom'#,'niveau_admin','lb_adm_tr' ], columns=['cd_type_statut'], aggfunc=list,fill_value=None) @@ -419,4 +295,8 @@ if __name__ == "__main__": ) # writer.save() print('pivot_libel OK !') - + typ.to_excel( + writer,sheet_name='dic_type_statut',index=False + ) + # writer.save() + print('dic_type_statut OK !') diff --git a/manage_bdd.py b/manage_bdd.py index b56158a..ddd1e46 100644 --- a/manage_bdd.py +++ b/manage_bdd.py @@ -26,7 +26,7 @@ def copy_2another_server(host_in,base_in,user_in,host_out,base_out,user_out,pass # pg_dump -C -h localhost -U localuser dbname | psql -h remotehost -U remoteuser dbname # pg_dump -C -h 172.17.0.2 -U postgres -d postgres --schema="07_202107" | psql -h 91.134.194.221 -U cgeier -d bd_cen # RESTORE AU NOM DE LA BDD CIBLE - # pg_dump -C -h 172.17.0.2 -U postgres -d postgres --schema="*_202207" --format=custom | pg_restore -h 91.134.194.221 -U cgeier --dbname="bd_cen" + # pg_dump -C -h 172.17.0.2 -U postgres -d postgres --schema="*_202207" --format=custom | pg_restore -h 91.134.194.221 -U cgeier --dbname="azalee_restore" pwd_in = 'export PGPASSWORD="{pwd}";'.format(pwd=passwd_in) cmd_in = 'pg_dump -C -h {H} -d {bdd} -U {U}'.format(