init depot renew
This commit is contained in:
@@ -0,0 +1,67 @@
|
||||
from pycen import con_gn
|
||||
from pycen.wfs import list_layer,get_wfs
|
||||
import geopandas as gpd
|
||||
|
||||
def extract_synthese_gn(table='v_synthese_for_export',con=con_gn,date_min=None,date_max=None,polygon=None,filter_obscen=True):
|
||||
sql = 'SELECT *,ST_GeomFromText(geometrie_wkt_4326,4326) geom FROM gn_synthese.{tab}'.format(tab=table)
|
||||
if any([i is not None for i in [date_min,date_max,polygon]]) or filter_obscen:
|
||||
sql += ' WHERE '
|
||||
if date_min is not None:
|
||||
sql += "date_debut > '%s' AND " % date_min
|
||||
if date_max is not None:
|
||||
sql += "date_debut < '%s' AND " % date_max
|
||||
if polygon is not None:
|
||||
sql += "ST_Intersects(ST_Transform(ST_GeomFromText(geometrie_wkt_4326,4326),2154),'SRID={epsg};{poly}') AND ".format(epsg=2154,poly=polygon)
|
||||
if filter_obscen:
|
||||
sql += """(
|
||||
unaccent(observateurs) LIKE ANY (array['%%Conservatoire d%%Espaces Naturels Isere%%', '%%Conservatoire d%%Espaces Naturels d%%Isere%%', '%%CEN Isere%%', '%%Association des Amis de l%%Ile de la Platiere%%'])
|
||||
OR observateurs NOT LIKE '%%(%%)%%'
|
||||
) AND observateurs NOT IN ('', 'Benoit Dodelin',', ','%%DELOCHE Denis%%')
|
||||
AND jdd_id NOT IN (185,377,236)
|
||||
AND ca_id NOT IN (53,65,66,1,2,6)
|
||||
"""
|
||||
|
||||
return gpd.read_postgis(sql,con_gn)
|
||||
|
||||
def format_forexport(df):
|
||||
is_date = df.columns.str.contains('date|heure',case=False,regex=True)
|
||||
is_uuid = df.columns.str.contains('uuid',case=False,regex=True)
|
||||
col_date = df.columns[is_uuid|is_date|(df.dtypes == 'datetime64[ns]')]
|
||||
df[col_date] = df[col_date].astype(str)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
url = 'http://ws.carmencarto.fr/WFS/94/ENS_CG38'
|
||||
list_layer(url)
|
||||
layers = ['zone_d_observation_01','Zone_d_observation_02']
|
||||
|
||||
gdf = gpd.pd.concat([
|
||||
get_wfs(url,layers[0]),
|
||||
get_wfs(url,layers[1])
|
||||
])
|
||||
|
||||
df_max20210720 = extract_synthese_gn(
|
||||
table='v_synthese_for_export',
|
||||
con=con_gn,
|
||||
date_max='2021-07-21',
|
||||
polygon=gdf.unary_union
|
||||
)
|
||||
|
||||
df_min20210721 = extract_synthese_gn(
|
||||
table='v_synthese_for_export',
|
||||
con=con_gn,
|
||||
date_min='2021-07-20',
|
||||
date_max='2024-01-01',
|
||||
polygon=gdf.unary_union
|
||||
)
|
||||
|
||||
|
||||
format_forexport(df_max20210720)
|
||||
format_forexport(df_min20210721)
|
||||
|
||||
df_max20210720.to_file('/home/colas/Documents/9_PROJETS/6_GEONATURE/EXPORT/export_for_dept.gpkg',driver='GPKG',layername='observations_max_20210720')
|
||||
df_min20210721.to_file('/home/colas/Documents/9_PROJETS/6_GEONATURE/EXPORT/export_for_dept.gpkg',driver='GPKG',layername='observations_min_20210721')
|
||||
|
||||
df_max20210720.drop('geom',axis=1).to_csv('/home/colas/Documents/9_PROJETS/6_GEONATURE/EXPORT/observationsENS_max_20210720.csv')
|
||||
df_min20210721.drop('geom',axis=1).to_csv('/home/colas/Documents/9_PROJETS/6_GEONATURE/EXPORT/observationsENS_min_20210721.csv')
|
||||
@@ -0,0 +1,213 @@
|
||||
from pycen import con_gn
|
||||
|
||||
sql = '''
|
||||
DROP VIEW IF EXISTS gn_exports.departement_export_ens_with_metadata;
|
||||
|
||||
CREATE VIEW gn_exports.departement_export_ens_with_metadata AS
|
||||
WITH af_actors AS (
|
||||
SELECT cafa.id_acquisition_framework,
|
||||
json_build_object('type_role',
|
||||
CASE
|
||||
WHEN cafa.id_organism IS NOT NULL THEN 'organism'::TEXT
|
||||
WHEN cafa.id_role IS NOT NULL THEN 'role'::TEXT
|
||||
ELSE NULL::TEXT
|
||||
END,
|
||||
'uuid_actor', coalesce(borg.uuid_organisme, tro.uuid_role),
|
||||
'cd_nomenclature_actor_role', tn.cd_nomenclature, 'identity',
|
||||
CASE
|
||||
WHEN cafa.id_organism IS NOT NULL
|
||||
THEN json_build_object('organism_name', borg.nom_organisme)
|
||||
WHEN cafa.id_role IS NOT NULL THEN json_build_object('first_name',
|
||||
tro.nom_role,
|
||||
'last_name',
|
||||
tro.prenom_role) END,
|
||||
'email', coalesce(borg.email_organisme, tro.email)) AS json_data
|
||||
FROM gn_meta.cor_acquisition_framework_actor cafa
|
||||
LEFT JOIN utilisateurs.bib_organismes borg ON cafa.id_organism = borg.id_organisme
|
||||
LEFT JOIN utilisateurs.t_roles tro ON cafa.id_role = tro.id_role
|
||||
JOIN ref_nomenclatures.t_nomenclatures tn
|
||||
ON cafa.id_nomenclature_actor_role = tn.id_nomenclature),
|
||||
af AS (
|
||||
SELECT taf.id_acquisition_framework,
|
||||
jsonb_build_object('uuid', taf.unique_acquisition_framework_id, 'name',
|
||||
taf.acquisition_framework_name,
|
||||
'desc', taf.acquisition_framework_desc, 'start_date',
|
||||
taf.acquisition_framework_start_date, 'end_date',
|
||||
taf.acquisition_framework_end_date,
|
||||
'initial_closing_date', taf.initial_closing_date, 'territorial_level',
|
||||
ntl.cd_nomenclature, 'financing_type', nft.cd_nomenclature, 'target_description',
|
||||
taf.target_description, 'ecologic_or_geologic_target',
|
||||
taf.ecologic_or_geologic_target, 'actors',
|
||||
json_agg(af_actors.json_data)) AS af_data
|
||||
FROM gn_meta.t_acquisition_frameworks taf
|
||||
JOIN af_actors ON af_actors.id_acquisition_framework = taf.id_acquisition_framework
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ntl
|
||||
ON taf.id_nomenclature_territorial_level = ntl.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures nft
|
||||
ON taf.id_nomenclature_financing_type = nft.id_nomenclature
|
||||
GROUP BY taf.id_acquisition_framework, taf.acquisition_framework_name, taf.acquisition_framework_desc,
|
||||
taf.acquisition_framework_start_date, taf.acquisition_framework_end_date, taf.initial_closing_date,
|
||||
ntl.cd_nomenclature, nft.cd_nomenclature),
|
||||
ds_actors AS (
|
||||
SELECT cda.id_dataset,
|
||||
json_build_object('type_role',
|
||||
CASE
|
||||
WHEN cda.id_organism IS NOT NULL THEN 'organism'::TEXT
|
||||
WHEN cda.id_role IS NOT NULL THEN 'role'::TEXT
|
||||
ELSE NULL::TEXT
|
||||
END, 'uuid_actor', coalesce(borg.uuid_organisme, tro.uuid_role),
|
||||
'cd_nomenclature_actor_role', tn.cd_nomenclature, 'identity',
|
||||
CASE
|
||||
WHEN cda.id_organism IS NOT NULL
|
||||
THEN json_build_object('organism_name', borg.nom_organisme)
|
||||
WHEN cda.id_role IS NOT NULL THEN json_build_object('first_name',
|
||||
tro.nom_role,
|
||||
'last_name',
|
||||
tro.prenom_role) END,
|
||||
'email', coalesce(borg.email_organisme, tro.email)) AS json_data
|
||||
FROM gn_meta.cor_dataset_actor cda
|
||||
LEFT JOIN utilisateurs.bib_organismes borg ON cda.id_organism = borg.id_organisme
|
||||
LEFT JOIN utilisateurs.t_roles tro ON cda.id_role = tro.id_role
|
||||
JOIN ref_nomenclatures.t_nomenclatures tn
|
||||
ON cda.id_nomenclature_actor_role = tn.id_nomenclature),
|
||||
ds AS (SELECT tds.id_dataset,
|
||||
tds.id_acquisition_framework,
|
||||
-- tds.additional_data,
|
||||
jsonb_build_object('uuid', tds.unique_dataset_id, 'name', tds.dataset_name, 'desc', tds.dataset_desc,
|
||||
'shortname', tds.dataset_shortname, 'data_type', ndt.cd_nomenclature,
|
||||
'collecting_method', ncm.cd_nomenclature, 'data_origin', ndo.cd_nomenclature,
|
||||
'dataset_objectif', ndso.cd_nomenclature, 'resource_type', nrt.cd_nomenclature,
|
||||
'source_status', nss.cd_nomenclature, 'territories', array_agg(DISTINCT
|
||||
ref_nomenclatures.get_cd_nomenclature(cdt.id_nomenclature_territory)),
|
||||
'actors', json_agg(ds_actors.json_data)) AS dataset_data
|
||||
FROM gn_meta.t_datasets tds
|
||||
JOIN ds_actors ON ds_actors.id_dataset = tds.id_dataset
|
||||
LEFT JOIN gn_meta.cor_dataset_territory cdt ON cdt.id_dataset = tds.id_dataset
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ndt
|
||||
ON tds.id_nomenclature_data_type = ndt.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ncm
|
||||
ON tds.id_nomenclature_collecting_method = ncm.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ndo
|
||||
ON tds.id_nomenclature_data_origin = ndo.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ndso
|
||||
ON tds.id_nomenclature_dataset_objectif = ndso.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures nrt
|
||||
ON tds.id_nomenclature_resource_type = nrt.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures nss
|
||||
ON tds.id_nomenclature_source_status = nss.id_nomenclature
|
||||
GROUP BY tds.id_dataset, tds.id_acquisition_framework, tds.unique_dataset_id, tds.dataset_name,
|
||||
tds.dataset_desc, tds.dataset_shortname, ndt.cd_nomenclature, ncm.cd_nomenclature,
|
||||
ndo.cd_nomenclature, ndso.cd_nomenclature, nrt.cd_nomenclature, nss.cd_nomenclature)
|
||||
SELECT s.id_synthese,
|
||||
s.entity_source_pk_value AS id_source,
|
||||
s.unique_id_sinp AS id_perm_sinp,
|
||||
s.unique_id_sinp_grp AS id_perm_grp_sinp,
|
||||
s.date_min AS date_debut,
|
||||
s.date_max AS date_fin,
|
||||
s.cd_nom,
|
||||
s.meta_v_taxref AS version_taxref,
|
||||
s.nom_cite,
|
||||
s.count_min AS nombre_min,
|
||||
s.count_max AS nombre_max,
|
||||
s.altitude_min,
|
||||
s.altitude_max,
|
||||
s.depth_min AS profondeur_min,
|
||||
s.depth_max AS profondeur_max,
|
||||
s.observers AS observateurs,
|
||||
s.determiner AS determinateur,
|
||||
s.validator AS validateur,
|
||||
s.sample_number_proof AS numero_preuve,
|
||||
s.digital_proof AS preuve_numerique,
|
||||
s.non_digital_proof AS preuve_non_numerique,
|
||||
s.comment_context AS comment_releve,
|
||||
s.comment_description AS comment_occurrence,
|
||||
ds.dataset_data AS jdd_data,
|
||||
af.af_data AS ca_data,
|
||||
s.reference_biblio,
|
||||
s.cd_hab AS code_habitat,
|
||||
h.lb_hab_fr AS habitat,
|
||||
s.place_name AS nom_lieu,
|
||||
s.precision,
|
||||
s.additional_data::TEXT AS donnees_additionnelles,
|
||||
s.the_geom_4326 AS geom,
|
||||
st_astext(s.the_geom_4326) AS wkt_4326,
|
||||
n1.cd_nomenclature AS nature_objet_geo,
|
||||
n2.cd_nomenclature AS type_regroupement,
|
||||
s.grp_method AS methode_regroupement,
|
||||
n3.cd_nomenclature AS comportement,
|
||||
n4.cd_nomenclature AS technique_obs,
|
||||
n5.cd_nomenclature AS statut_biologique,
|
||||
n6.cd_nomenclature AS etat_biologique,
|
||||
n7.cd_nomenclature AS naturalite,
|
||||
n8.cd_nomenclature AS preuve_existante,
|
||||
n9.cd_nomenclature AS precision_diffusion,
|
||||
n10.cd_nomenclature AS stade_vie,
|
||||
n11.cd_nomenclature AS sexe,
|
||||
n12.cd_nomenclature AS objet_denombrement,
|
||||
n13.cd_nomenclature AS type_denombrement,
|
||||
n14.cd_nomenclature AS niveau_sensibilite,
|
||||
n15.cd_nomenclature AS statut_observation,
|
||||
n16.cd_nomenclature AS floutage_dee,
|
||||
n17.cd_nomenclature AS statut_source,
|
||||
n18.cd_nomenclature AS type_info_geo,
|
||||
n19.cd_nomenclature AS methode_determination,
|
||||
n20.cd_nomenclature AS statut_validation,
|
||||
coalesce(s.meta_update_date, s.meta_create_date) AS derniere_action
|
||||
FROM gn_synthese.synthese s
|
||||
JOIN ds ON ds.id_dataset = s.id_dataset
|
||||
JOIN af ON ds.id_acquisition_framework = af.id_acquisition_framework
|
||||
LEFT JOIN ref_habitats.habref h ON h.cd_hab = s.cd_hab
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n1 ON s.id_nomenclature_geo_object_nature = n1.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n2 ON s.id_nomenclature_grp_typ = n2.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n3 ON s.id_nomenclature_behaviour = n3.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n4 ON s.id_nomenclature_obs_technique = n4.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n5 ON s.id_nomenclature_bio_status = n5.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n6 ON s.id_nomenclature_bio_condition = n6.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n7 ON s.id_nomenclature_naturalness = n7.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n8 ON s.id_nomenclature_exist_proof = n8.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n9 ON s.id_nomenclature_diffusion_level = n9.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n10 ON s.id_nomenclature_life_stage = n10.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n11 ON s.id_nomenclature_sex = n11.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n12 ON s.id_nomenclature_obj_count = n12.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n13 ON s.id_nomenclature_type_count = n13.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n14 ON s.id_nomenclature_sensitivity = n14.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n15 ON s.id_nomenclature_observation_status = n15.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n16 ON s.id_nomenclature_blurring = n16.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n17 ON s.id_nomenclature_source_status = n17.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n18 ON s.id_nomenclature_info_geo_type = n18.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n19 ON s.id_nomenclature_determination_method = n19.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n20 ON s.id_nomenclature_valid_status = n20.id_nomenclature
|
||||
|
||||
JOIN taxonomie.taxref t ON t.cd_nom = s.cd_nom
|
||||
JOIN gn_synthese.t_sources ts ON ts.id_source = s.id_source
|
||||
|
||||
WHERE ST_INTERSECTS(
|
||||
s.the_geom_4326,
|
||||
(SELECT ST_TRANSFORM(ST_UNION(geom),4326) FROM ref_geo.l_areas
|
||||
JOIN ref_geo.bib_areas_types USING (id_type)
|
||||
WHERE type_code IN ('ENS_ZO','ENS_ZI')
|
||||
--AND area_code IN ('SD30','SD29','SD28','SD26','SD24','SD19','SD18','SD17','SD16','SD15','SD12','SD11','SD09','SD06',z'SD05','SD02','SD01','PSND2','PSND1')
|
||||
))
|
||||
-- exclision CA ([SICEN] Données anciennes,[TEST] : migration sicen,[SICEN] Hors Etude)
|
||||
AND ds.id_acquisition_framework NOT IN (53,65,66,1,2,6)
|
||||
-- exclision JDD `Observations opportunistes du CEN Isère importé depuis Faune Isère`
|
||||
AND ds.id_dataset NOT IN (185,377,236)
|
||||
-- exclision CA (`Gestion de l'Espace Naturel Sensible du Méandre des Oves`, `Gestion de la Réserve Naturelle Nationale de l'Ile de la Platière`, `Gestion des Natura 2000 FR 8201749 « Milieux alluviaux et aquatiques de l’Île de la Platière » et FR 8212012 « Île de la Platière »`, `RNN Platière`
|
||||
--AND (ts.id_module = 7 AND ds.id_acquisition_framework IN (7,38,39,42,44,45,46,48,55,57,58,60)
|
||||
AND (
|
||||
unaccent(observers) LIKE ANY (array['%%Conservatoire d%%Espaces Naturels Isere%%', '%%Conservatoire d%%Espaces Naturels d%%Isere%%', '%%CEN Isere%%', '%%Association des Amis de l%%Ile de la Platiere%%'])
|
||||
OR observers NOT LIKE '%%(%%)%%'
|
||||
)
|
||||
AND unaccent(observers) NOT LIKE ALL(array[
|
||||
'%%Sympetrum%%','%%Departement%%','%%Rhone%%','%%Oxalis%%',
|
||||
'%%LPO%%','%%GRPLS%%','%%Parvi%%','%%CD Isere%%','%%Personnel%%','%%Independant%%'
|
||||
'%%DELOCHE Denis%%'
|
||||
])
|
||||
AND observers NOT IN ('', 'Benoit Dodelin',', ')
|
||||
-- statut_validation = 'Certain - très probable'
|
||||
-- AND n20.cd_nomenclature = '1'
|
||||
ORDER BY s.id_synthese;
|
||||
'''
|
||||
|
||||
with con_gn.begin() as cnx:
|
||||
cnx.execute(sql)
|
||||
@@ -0,0 +1,149 @@
|
||||
from pycen import con_gn
|
||||
|
||||
sql = '''
|
||||
DROP VIEW IF EXISTS gn_exports.export_taxon_unique;
|
||||
|
||||
CREATE VIEW gn_exports.export_taxon_unique AS
|
||||
WITH af_actors AS (
|
||||
SELECT cafa.id_acquisition_framework,
|
||||
json_build_object('type_role',
|
||||
CASE
|
||||
WHEN cafa.id_organism IS NOT NULL THEN 'organism'::TEXT
|
||||
WHEN cafa.id_role IS NOT NULL THEN 'role'::TEXT
|
||||
ELSE NULL::TEXT
|
||||
END,
|
||||
'uuid_actor', coalesce(borg.uuid_organisme, tro.uuid_role),
|
||||
'cd_nomenclature_actor_role', tn.cd_nomenclature, 'identity',
|
||||
CASE
|
||||
WHEN cafa.id_organism IS NOT NULL
|
||||
THEN json_build_object('organism_name', borg.nom_organisme)
|
||||
WHEN cafa.id_role IS NOT NULL THEN json_build_object('first_name',
|
||||
tro.nom_role,
|
||||
'last_name',
|
||||
tro.prenom_role) END,
|
||||
'email', coalesce(borg.email_organisme, tro.email)) AS json_data
|
||||
FROM gn_meta.cor_acquisition_framework_actor cafa
|
||||
LEFT JOIN utilisateurs.bib_organismes borg ON cafa.id_organism = borg.id_organisme
|
||||
LEFT JOIN utilisateurs.t_roles tro ON cafa.id_role = tro.id_role
|
||||
JOIN ref_nomenclatures.t_nomenclatures tn
|
||||
ON cafa.id_nomenclature_actor_role = tn.id_nomenclature),
|
||||
af AS (
|
||||
SELECT taf.id_acquisition_framework,
|
||||
jsonb_build_object('uuid', taf.unique_acquisition_framework_id, 'name',
|
||||
taf.acquisition_framework_name,
|
||||
'desc', taf.acquisition_framework_desc, 'start_date',
|
||||
taf.acquisition_framework_start_date, 'end_date',
|
||||
taf.acquisition_framework_end_date,
|
||||
'initial_closing_date', taf.initial_closing_date, 'territorial_level',
|
||||
ntl.cd_nomenclature, 'financing_type', nft.cd_nomenclature, 'target_description',
|
||||
taf.target_description, 'ecologic_or_geologic_target',
|
||||
taf.ecologic_or_geologic_target, 'actors',
|
||||
json_agg(af_actors.json_data)) AS af_data
|
||||
FROM gn_meta.t_acquisition_frameworks taf
|
||||
JOIN af_actors ON af_actors.id_acquisition_framework = taf.id_acquisition_framework
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ntl
|
||||
ON taf.id_nomenclature_territorial_level = ntl.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures nft
|
||||
ON taf.id_nomenclature_financing_type = nft.id_nomenclature
|
||||
GROUP BY taf.id_acquisition_framework, taf.acquisition_framework_name, taf.acquisition_framework_desc,
|
||||
taf.acquisition_framework_start_date, taf.acquisition_framework_end_date, taf.initial_closing_date,
|
||||
ntl.cd_nomenclature, nft.cd_nomenclature),
|
||||
ds_actors AS (
|
||||
SELECT cda.id_dataset,
|
||||
json_build_object('type_role',
|
||||
CASE
|
||||
WHEN cda.id_organism IS NOT NULL THEN 'organism'::TEXT
|
||||
WHEN cda.id_role IS NOT NULL THEN 'role'::TEXT
|
||||
ELSE NULL::TEXT
|
||||
END, 'uuid_actor', coalesce(borg.uuid_organisme, tro.uuid_role),
|
||||
'cd_nomenclature_actor_role', tn.cd_nomenclature, 'identity',
|
||||
CASE
|
||||
WHEN cda.id_organism IS NOT NULL
|
||||
THEN json_build_object('organism_name', borg.nom_organisme)
|
||||
WHEN cda.id_role IS NOT NULL THEN json_build_object('first_name',
|
||||
tro.nom_role,
|
||||
'last_name',
|
||||
tro.prenom_role) END,
|
||||
'email', coalesce(borg.email_organisme, tro.email)) AS json_data
|
||||
FROM gn_meta.cor_dataset_actor cda
|
||||
LEFT JOIN utilisateurs.bib_organismes borg ON cda.id_organism = borg.id_organisme
|
||||
LEFT JOIN utilisateurs.t_roles tro ON cda.id_role = tro.id_role
|
||||
JOIN ref_nomenclatures.t_nomenclatures tn
|
||||
ON cda.id_nomenclature_actor_role = tn.id_nomenclature),
|
||||
ds AS (SELECT tds.id_dataset,
|
||||
tds.id_acquisition_framework,
|
||||
-- tds.additional_data,
|
||||
jsonb_build_object('uuid', tds.unique_dataset_id, 'name', tds.dataset_name, 'desc', tds.dataset_desc,
|
||||
'shortname', tds.dataset_shortname, 'data_type', ndt.cd_nomenclature,
|
||||
'collecting_method', ncm.cd_nomenclature, 'data_origin', ndo.cd_nomenclature,
|
||||
'dataset_objectif', ndso.cd_nomenclature, 'resource_type', nrt.cd_nomenclature,
|
||||
'source_status', nss.cd_nomenclature, 'territories', array_agg(DISTINCT
|
||||
ref_nomenclatures.get_cd_nomenclature(cdt.id_nomenclature_territory)),
|
||||
'actors', json_agg(ds_actors.json_data)) AS dataset_data
|
||||
FROM gn_meta.t_datasets tds
|
||||
JOIN ds_actors ON ds_actors.id_dataset = tds.id_dataset
|
||||
LEFT JOIN gn_meta.cor_dataset_territory cdt ON cdt.id_dataset = tds.id_dataset
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ndt
|
||||
ON tds.id_nomenclature_data_type = ndt.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ncm
|
||||
ON tds.id_nomenclature_collecting_method = ncm.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ndo
|
||||
ON tds.id_nomenclature_data_origin = ndo.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ndso
|
||||
ON tds.id_nomenclature_dataset_objectif = ndso.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures nrt
|
||||
ON tds.id_nomenclature_resource_type = nrt.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures nss
|
||||
ON tds.id_nomenclature_source_status = nss.id_nomenclature
|
||||
GROUP BY tds.id_dataset, tds.id_acquisition_framework, tds.unique_dataset_id, tds.dataset_name,
|
||||
tds.dataset_desc, tds.dataset_shortname, ndt.cd_nomenclature, ncm.cd_nomenclature,
|
||||
ndo.cd_nomenclature, ndso.cd_nomenclature, nrt.cd_nomenclature, nss.cd_nomenclature)
|
||||
SELECT DISTINCT ON (s.cd_nom)
|
||||
s.cd_nom,
|
||||
s.nom_cite,
|
||||
t.regne,
|
||||
t.classe,
|
||||
t.ordre,
|
||||
t.famille,
|
||||
t.sous_famille,
|
||||
t.cd_ref,
|
||||
t.lb_nom,
|
||||
t.nom_complet,
|
||||
t.nom_vern,
|
||||
t.group1_inpn,
|
||||
t.group2_inpn,
|
||||
t.group3_inpn,
|
||||
count(s.cd_nom) occurence
|
||||
FROM gn_synthese.synthese s
|
||||
JOIN ds ON ds.id_dataset = s.id_dataset
|
||||
JOIN af ON ds.id_acquisition_framework = af.id_acquisition_framework
|
||||
LEFT JOIN ref_habitats.habref h ON h.cd_hab = s.cd_hab
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n1 ON s.id_nomenclature_geo_object_nature = n1.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n2 ON s.id_nomenclature_grp_typ = n2.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n3 ON s.id_nomenclature_behaviour = n3.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n4 ON s.id_nomenclature_obs_technique = n4.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n5 ON s.id_nomenclature_bio_status = n5.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n6 ON s.id_nomenclature_bio_condition = n6.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n7 ON s.id_nomenclature_naturalness = n7.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n8 ON s.id_nomenclature_exist_proof = n8.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n9 ON s.id_nomenclature_diffusion_level = n9.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n10 ON s.id_nomenclature_life_stage = n10.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n11 ON s.id_nomenclature_sex = n11.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n12 ON s.id_nomenclature_obj_count = n12.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n13 ON s.id_nomenclature_type_count = n13.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n14 ON s.id_nomenclature_sensitivity = n14.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n15 ON s.id_nomenclature_observation_status = n15.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n16 ON s.id_nomenclature_blurring = n16.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n17 ON s.id_nomenclature_source_status = n17.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n18 ON s.id_nomenclature_info_geo_type = n18.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n19 ON s.id_nomenclature_determination_method = n19.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n20 ON s.id_nomenclature_valid_status = n20.id_nomenclature
|
||||
|
||||
JOIN taxonomie.taxref t ON t.cd_nom = s.cd_nom
|
||||
JOIN gn_synthese.t_sources ts ON ts.id_source = s.id_source
|
||||
Group by s.cd_nom, s.nom_cite, t.regne, t.classe, t.ordre, t.famille, t.sous_famille, t.cd_ref, t.lb_nom, t.nom_complet, t.nom_vern, t.group1_inpn, t.group2_inpn, t.group3_inpn
|
||||
;
|
||||
'''
|
||||
|
||||
with con_gn.begin() as cnx:
|
||||
cnx.execute(sql)
|
||||
@@ -0,0 +1,186 @@
|
||||
# SCRIPT NON VALIDE À CE JOUR
|
||||
from pycen import con_gn
|
||||
|
||||
sql = '''
|
||||
DROP VIEW IF EXISTS gn_exports.v_synthese_sinp_with_metadata_flora_for_gn2pg_bis;
|
||||
|
||||
CREATE VIEW gn_exports.v_synthese_sinp_with_metadata_flora_for_gn2pg_bis AS
|
||||
WITH af_actors AS (
|
||||
SELECT cafa.id_acquisition_framework
|
||||
FROM gn_meta.cor_acquisition_framework_actor cafa
|
||||
LEFT JOIN utilisateurs.bib_organismes borg ON cafa.id_organism = borg.id_organisme
|
||||
WHERE borg.uuid_organisme = '5a433bd0-2070-25d9-e053-2614a8c026f8'::uuid
|
||||
), af AS (
|
||||
SELECT taf.id_acquisition_framework
|
||||
FROM gn_meta.t_acquisition_frameworks taf
|
||||
JOIN af_actors ON af_actors.id_acquisition_framework = taf.id_acquisition_framework
|
||||
GROUP BY taf.id_acquisition_framework
|
||||
), ds AS (
|
||||
SELECT tds.id_dataset,
|
||||
tds.dataset_name AS nom_jdd,
|
||||
tds.id_acquisition_framework
|
||||
FROM gn_meta.t_datasets tds
|
||||
GROUP BY tds.id_dataset, tds.dataset_name
|
||||
), geo AS (
|
||||
SELECT "left"(geo_1.area_code::text, 2) AS departement,
|
||||
geo_1.area_code AS commune,
|
||||
s_1.id_synthese,
|
||||
st_transform(s_1.the_geom_local, 4326) AS st_transform
|
||||
FROM ref_geo.l_areas geo_1
|
||||
JOIN gn_synthese.synthese s_1 ON st_intersects(s_1.the_geom_4326, st_transform(geo_1.geom, 4326))
|
||||
WHERE geo_1.id_type = 25
|
||||
)
|
||||
SELECT
|
||||
ds.nom_jdd AS nom_jdd,
|
||||
s.unique_id_sinp_grp AS id_sinp_releve,
|
||||
occ.id_releve_occtax AS identifiant_releve,
|
||||
NULL::text AS code_perso_releve,
|
||||
s.unique_id_sinp AS id_sinp_observation,
|
||||
s.entity_source_pk_value AS identifiant_observation,
|
||||
geo.departement,
|
||||
geo.commune,
|
||||
NULL::text AS lieu_dit,
|
||||
CASE
|
||||
WHEN "position"(sp.srtext::text, 'GEOGCS'::text) = 1 THEN "substring"(replace(sp.srtext::text, 'GEOGCS["'::text, ''::text), 1, "position"(replace(sp.srtext::text, 'GEOGCS["'::text, ''::text), '",'::text) - 1)
|
||||
WHEN "position"(sp.srtext::text, 'PROJCS'::text) = 1 THEN "substring"(replace(sp.srtext::text, 'PROJCS["'::text, ''::text), 1, "position"(replace(sp.srtext::text, 'PROJCS["'::text, ''::text), '",'::text) - 1)
|
||||
WHEN "position"(sp.srtext::text, 'GEOCCS'::text) = 1 THEN "substring"(replace(sp.srtext::text, 'GEOCCS["'::text, ''::text), 1, "position"(replace(sp.srtext::text, 'GEOCCS["'::text, ''::text), '",'::text) - 1)
|
||||
WHEN "position"(sp.srtext::text, 'COMPD_CS'::text) = 1 THEN "substring"(replace(sp.srtext::text, 'COMPD_CS["'::text, ''::text), 1, "position"(replace(sp.srtext::text, 'COMPD_CS["'::text, ''::text), '",'::text) - 1)
|
||||
ELSE 'Non défini'::text
|
||||
END AS sys_coord,
|
||||
st_srid(s.the_geom_4326) AS "SRID",
|
||||
st_astext(s.the_geom_4326) AS localisation_wkt,
|
||||
st_x(st_centroid(s.the_geom_4326)) AS coord_x,
|
||||
st_y(st_centroid(s.the_geom_4326)) AS coord_y,
|
||||
s."precision" AS precision,
|
||||
NULL::text AS nature_objet,
|
||||
s.altitude_min AS alti_min,
|
||||
s.altitude_max AS alti_max,
|
||||
NULL::text AS pente,
|
||||
NULL::text AS exposition,
|
||||
NULL::text AS comm_geol,
|
||||
NULL::text AS milieu,
|
||||
s.observers AS observateurs,
|
||||
s.date_min AS date_debut,
|
||||
s.date_max AS date_fin,
|
||||
NULL::text AS comm_context,
|
||||
n2.mnemonique AS type_regroupement,
|
||||
s.grp_method AS meth_regroupement,
|
||||
NULL::text AS surface,
|
||||
(s."additional_data"::json#>>'{strate_flore}') ||
|
||||
ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)
|
||||
AS strate_vegetation,
|
||||
(s."additional_data"::json#>>'{heigth_herbace}')::text || (s."additional_data"::json#>>'{heigth_arbust}')::text
|
||||
|| (s."additional_data"::json#>>'{heigth_arbore}')::text || (s."additional_data"::json#>>'{heigth_ssarbust}')::text
|
||||
AS hauteur_strate,
|
||||
(s."additional_data"::json#>>'{rcvmt_herbace}')::text || (s."additional_data"::json#>>'{rcvmt_arbust}')::text
|
||||
|| (s."additional_data"::json#>>'{rcvmt_arbore}')::text || (s."additional_data"::json#>>'{rcvmt_ssarbust}')::text
|
||||
AS recouvrement_strate,
|
||||
s.cd_hab AS cdhab,
|
||||
NULL::text AS cdhab_v,
|
||||
NULL::text AS code_eur,
|
||||
NULL::text AS code_eunis,
|
||||
NULL::text AS code_cahab,
|
||||
NULL::text AS code_cb,
|
||||
NULL::text AS id_microhab,
|
||||
n21.regne AS regne,
|
||||
s.nom_cite AS nom_cite,
|
||||
s.cd_nom as cd_nom,
|
||||
(s."additional_data"::json#>>'{effectif_textuel}')::text ||
|
||||
ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_braunblanq_abdom}')::int)::text
|
||||
AS abondance,
|
||||
NULL::text AS sociabilite,
|
||||
n11.mnemonique AS sexe,
|
||||
n7.mnemonique AS naturalite,
|
||||
NULL::text AS comm_description,
|
||||
n15.mnemonique AS statut_observation,
|
||||
n12.mnemonique AS objet_denombrement,
|
||||
n13.mnemonique AS type_denombrement,
|
||||
s.count_min AS nombre_min,
|
||||
s.count_max AS nombre_max,
|
||||
n17.label_default AS statut_source,
|
||||
NULL::text AS reference_biblio,
|
||||
NULL::text AS page,
|
||||
n8.mnemonique AS preuve_existence,
|
||||
s.digital_proof AS preuve_numerique,
|
||||
s.non_digital_proof AS preuve_non_numerique,
|
||||
NULL::text AS nom_collection,
|
||||
NULL::text AS ref_collection,
|
||||
s.determiner AS determinateur,
|
||||
NULL::text AS niv_val,
|
||||
NULL::text AS niveau_diffusion,
|
||||
n16.mnemonique AS floutage_dee,
|
||||
NULL::text AS methode_observation,
|
||||
CASE WHEN ts.name_source like 'MONITORING_%%'
|
||||
THEN split_part(ts.name_source,'_',2)
|
||||
ELSE 'RELEVE FLORE'::text
|
||||
END AS protocole,
|
||||
n6.mnemonique AS etat_biologique,
|
||||
n5.mnemonique AS statut_biologique,
|
||||
n10.mnemonique AS stade_vie,
|
||||
n19.mnemonique AS methode_determination,
|
||||
n3.mnemonique AS comportement,
|
||||
s.additional_data - '{effectif_textuel,strate_flore,code_atlas,id_nomenclature_braunblanq_abdom,id_nomenclature_strate_flore}'::text[] AS donnees_additionnelles
|
||||
FROM gn_synthese.synthese s
|
||||
JOIN ds ON ds.id_dataset = s.id_dataset
|
||||
JOIN af ON ds.id_acquisition_framework = af.id_acquisition_framework
|
||||
JOIN geo ON s.id_synthese = geo.id_synthese
|
||||
JOIN spatial_ref_sys sp ON st_srid(s.the_geom_4326) = sp.auth_srid
|
||||
LEFT JOIN ref_habitats.habref h ON h.cd_hab = s.cd_hab
|
||||
LEFT JOIN pr_occtax.t_releves_occtax occ ON occ.unique_id_sinp_grp = s.unique_id_sinp_grp
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n1 ON s.id_nomenclature_geo_object_nature = n1.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n2 ON s.id_nomenclature_grp_typ = n2.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n3 ON s.id_nomenclature_behaviour = n3.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n4 ON s.id_nomenclature_obs_technique = n4.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n5 ON s.id_nomenclature_bio_status = n5.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n6 ON s.id_nomenclature_bio_condition = n6.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n7 ON s.id_nomenclature_naturalness = n7.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n8 ON s.id_nomenclature_exist_proof = n8.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n9 ON s.id_nomenclature_diffusion_level = n9.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n10 ON s.id_nomenclature_life_stage = n10.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n11 ON s.id_nomenclature_sex = n11.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n12 ON s.id_nomenclature_obj_count = n12.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n13 ON s.id_nomenclature_type_count = n13.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n14 ON s.id_nomenclature_sensitivity = n14.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n15 ON s.id_nomenclature_observation_status = n15.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n16 ON s.id_nomenclature_blurring = n16.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n17 ON s.id_nomenclature_source_status = n17.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n18 ON s.id_nomenclature_info_geo_type = n18.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n19 ON s.id_nomenclature_determination_method = n19.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n20 ON s.id_nomenclature_valid_status = n20.id_nomenclature
|
||||
LEFT JOIN taxonomie.taxref n21 ON s.cd_nom = n21.cd_nom
|
||||
JOIN gn_synthese.cor_area_synthese cas ON cas.id_synthese = s.id_synthese
|
||||
|
||||
-- ADD CEN_38
|
||||
JOIN gn_synthese.t_sources ts ON ts.id_source = s.id_source
|
||||
WHERE n21.regne::text != 'Animalia'::text
|
||||
-- exclision CA ([SICEN] Données anciennes,[TEST] : migration sicen,[SICEN] Hors Etude)
|
||||
AND ds.id_acquisition_framework NOT IN (
|
||||
53,1,2,6, -- CA tests
|
||||
84, -- Echange de données PG Chravas
|
||||
85, -- Echange de données
|
||||
116 -- Echange de données issus de SICEN
|
||||
)
|
||||
-- exclision JDD `Observations opportunistes du CEN Isère importé depuis Faune Isère`
|
||||
AND ds.id_dataset NOT IN (
|
||||
185,377,236,
|
||||
409 -- Echange de données CORA
|
||||
)
|
||||
-- exclision CA (`Gestion de l'Espace Naturel Sensible du Méandre des Oves`, `Gestion de la Réserve Naturelle Nationale de l'Ile de la Platière`, `Gestion des Natura 2000 FR 8201749 « Milieux alluviaux et aquatiques de l’Île de la Platière » et FR 8212012 « Île de la Platière »`, `RNN Platière`
|
||||
--AND (ts.id_module = 7 AND ds.id_acquisition_framework IN (7,38,39,42,44,45,46,48,55,57,58,60)
|
||||
AND (
|
||||
unaccent(observers) LIKE ANY (array['%%Conservatoire d%%Espaces Naturels Isere%%', '%%Conservatoire d%%Espaces Naturels d%%Isere%%', '%%CEN Isere%%', '%%Association des Amis de l%%Ile de la Platiere%%'])
|
||||
OR observers NOT LIKE '%%(%%)%%'
|
||||
)
|
||||
AND unaccent(observers) NOT LIKE ALL(array[
|
||||
'%%Sympetrum%%','%%Departement%%','%%Rhone%%','%%Oxalis%%',
|
||||
'%%LPO%%','%%GRPLS%%','%%Parvi%%','%%CD Isere%%','%%Personnel%%','%%Independant%%'
|
||||
'%%DELOCHE Denis%%'
|
||||
])
|
||||
AND observers NOT IN ('', 'Benoit Dodelin',', ')
|
||||
-- statut_validation = 'Certain - très probable'
|
||||
-- AND n20.cd_nomenclature = '1'
|
||||
ORDER BY s.id_synthese;
|
||||
'''
|
||||
|
||||
with con_gn.begin() as cnx:
|
||||
cnx.execute(sql)
|
||||
@@ -0,0 +1,203 @@
|
||||
from pycen import con_gn
|
||||
|
||||
sql = '''
|
||||
DROP VIEW IF EXISTS gn_exports.v_synthese_sinp_with_metadata_flora_for_gn2pg CASCADE;
|
||||
|
||||
CREATE VIEW gn_exports.v_synthese_sinp_with_metadata_flora_for_gn2pg AS
|
||||
WITH af_actors AS (
|
||||
SELECT cafa.id_acquisition_framework
|
||||
FROM gn_meta.cor_acquisition_framework_actor cafa
|
||||
LEFT JOIN utilisateurs.bib_organismes borg ON cafa.id_organism = borg.id_organisme
|
||||
WHERE borg.uuid_organisme = '5a433bd0-2070-25d9-e053-2614a8c026f8'::uuid
|
||||
), af AS (
|
||||
SELECT taf.id_acquisition_framework, unique_acquisition_framework_id
|
||||
FROM gn_meta.t_acquisition_frameworks taf
|
||||
JOIN af_actors ON af_actors.id_acquisition_framework = taf.id_acquisition_framework
|
||||
GROUP BY taf.id_acquisition_framework
|
||||
), ds AS (
|
||||
SELECT tds.id_dataset, tds.unique_dataset_id,
|
||||
tds.dataset_name AS nom_jdd,
|
||||
tds.id_acquisition_framework
|
||||
FROM gn_meta.t_datasets tds
|
||||
GROUP BY tds.id_dataset, tds.dataset_name
|
||||
), geo AS (
|
||||
SELECT "left"(geo_1.area_code::text, 2) AS departement,
|
||||
json_agg(geo_1.area_code) AS commune,
|
||||
s_1.id_synthese,
|
||||
st_transform(s_1.the_geom_local, 4326) AS st_transform
|
||||
FROM ref_geo.l_areas geo_1
|
||||
JOIN gn_synthese.cor_area_synthese USING (id_area)
|
||||
JOIN gn_synthese.synthese s_1 USING (id_synthese)
|
||||
WHERE geo_1.id_type = 25
|
||||
GROUP BY 1,3,4
|
||||
)
|
||||
SELECT
|
||||
af.unique_acquisition_framework_id id_sinp_ca,
|
||||
ds.nom_jdd AS nom_jdd,
|
||||
ds.unique_dataset_id id_sinp_jdd,
|
||||
s.unique_id_sinp_grp AS id_sinp_releve,
|
||||
occ.id_releve_occtax AS identifiant_releve,
|
||||
NULL::text AS code_perso_releve,
|
||||
s.unique_id_sinp AS id_sinp_observation,
|
||||
s.entity_source_pk_value AS identifiant_observation,
|
||||
geo.departement,
|
||||
geo.commune,
|
||||
NULL::text AS lieu_dit,
|
||||
CASE
|
||||
WHEN "position"(sp.srtext::text, 'GEOGCS'::text) = 1 THEN "substring"(replace(sp.srtext::text, 'GEOGCS["'::text, ''::text), 1, "position"(replace(sp.srtext::text, 'GEOGCS["'::text, ''::text), '",'::text) - 1)
|
||||
WHEN "position"(sp.srtext::text, 'PROJCS'::text) = 1 THEN "substring"(replace(sp.srtext::text, 'PROJCS["'::text, ''::text), 1, "position"(replace(sp.srtext::text, 'PROJCS["'::text, ''::text), '",'::text) - 1)
|
||||
WHEN "position"(sp.srtext::text, 'GEOCCS'::text) = 1 THEN "substring"(replace(sp.srtext::text, 'GEOCCS["'::text, ''::text), 1, "position"(replace(sp.srtext::text, 'GEOCCS["'::text, ''::text), '",'::text) - 1)
|
||||
WHEN "position"(sp.srtext::text, 'COMPD_CS'::text) = 1 THEN "substring"(replace(sp.srtext::text, 'COMPD_CS["'::text, ''::text), 1, "position"(replace(sp.srtext::text, 'COMPD_CS["'::text, ''::text), '",'::text) - 1)
|
||||
ELSE 'Non défini'::text
|
||||
END AS sys_coord,
|
||||
st_srid(s.the_geom_4326) AS "SRID",
|
||||
st_astext(s.the_geom_4326) AS localisation_wkt,
|
||||
st_x(st_centroid(s.the_geom_4326)) AS coord_x,
|
||||
st_y(st_centroid(s.the_geom_4326)) AS coord_y,
|
||||
s."precision" AS precision,
|
||||
NULL::text AS nature_objet,
|
||||
s.altitude_min AS alti_min,
|
||||
s.altitude_max AS alti_max,
|
||||
NULL::text AS pente,
|
||||
NULL::text AS exposition,
|
||||
NULL::text AS comm_geol,
|
||||
NULL::text AS milieu,
|
||||
s.observers AS observateurs,
|
||||
s.date_min AS date_debut,
|
||||
s.date_max AS date_fin,
|
||||
NULL::text AS comm_context,
|
||||
n2.mnemonique AS type_regroupement,
|
||||
s.grp_method AS meth_regroupement,
|
||||
NULL::text AS surface,
|
||||
CASE WHEN (s."additional_data"::json#>>'{strate_flore}') IS NOT NULL
|
||||
THEN (s."additional_data"::json#>>'{strate_flore}')::text
|
||||
WHEN s."additional_data"::json#>>'{id_nomenclature_strate_flore}' IS NOT NULL
|
||||
THEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text
|
||||
ELSE NULL::text
|
||||
END AS strate_vegetation,
|
||||
CASE WHEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text = 'Strate herbacée'::text
|
||||
THEN (s."additional_data"::json#>>'{heigth_herbace}')::text
|
||||
WHEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text = 'Strate arbustive'::text
|
||||
THEN (s."additional_data"::json#>>'{heigth_arbust}')::text
|
||||
WHEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text = 'Strate arborée'::text
|
||||
THEN (s."additional_data"::json#>>'{heigth_arbore}')::text
|
||||
WHEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text = 'Strate sous-arbustive'::text
|
||||
THEN (s."additional_data"::json#>>'{heigth_ssarbust}')::text
|
||||
ELSE NULL::text
|
||||
END AS hauteur_strate,
|
||||
CASE WHEN s."additional_data"::json#>>'{id_nomenclature_strate_flore}' IS NOT NULL
|
||||
THEN CASE WHEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text = 'Strate herbacée'::text
|
||||
THEN (s."additional_data"::json#>>'{rcvmt_herbace}')::text
|
||||
WHEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text = 'Strate arbustive'::text
|
||||
THEN (s."additional_data"::json#>>'{rcvmt_arbust}')::text
|
||||
WHEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text = 'Strate arborée'::text
|
||||
THEN (s."additional_data"::json#>>'{rcvmt_arbore}')::text
|
||||
WHEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text = 'Strate sous-arbustive'::text
|
||||
THEN (s."additional_data"::json#>>'{rcvmt_ssarbust}')::text
|
||||
ELSE NULL::text END
|
||||
ELSE NULL::text
|
||||
END AS recouvrement_strate,
|
||||
s.cd_hab AS cdhab,
|
||||
NULL::text AS cdhab_v,
|
||||
NULL::text AS code_eur,
|
||||
NULL::text AS code_eunis,
|
||||
NULL::text AS code_cahab,
|
||||
NULL::text AS code_cb,
|
||||
NULL::text AS id_microhab,
|
||||
n21.regne AS regne,
|
||||
s.nom_cite AS nom_cite,
|
||||
s.cd_nom as cd_nom,
|
||||
CASE WHEN (s."additional_data"::json#>>'{effectif_textuel}') IS NOT NULL
|
||||
THEN (s."additional_data"::json#>>'{effectif_textuel}')::text
|
||||
WHEN s."additional_data"::json#>>'{id_nomenclature_braunblanq_abdom}' IS NOT NULL
|
||||
THEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_braunblanq_abdom}')::int)::text
|
||||
ELSE NULL::text
|
||||
END AS abondance,
|
||||
NULL::text AS sociabilite,
|
||||
n11.mnemonique AS sexe,
|
||||
n7.mnemonique AS naturalite,
|
||||
NULL::text AS comm_description,
|
||||
n15.mnemonique AS statut_observation,
|
||||
n12.mnemonique AS objet_denombrement,
|
||||
n13.mnemonique AS type_denombrement,
|
||||
s.count_min AS nombre_min,
|
||||
s.count_max AS nombre_max,
|
||||
n17.label_default AS statut_source,
|
||||
NULL::text AS reference_biblio,
|
||||
NULL::text AS page,
|
||||
n8.mnemonique AS preuve_existence,
|
||||
s.digital_proof AS preuve_numerique,
|
||||
s.non_digital_proof AS preuve_non_numerique,
|
||||
NULL::text AS nom_collection,
|
||||
NULL::text AS ref_collection,
|
||||
s.determiner AS determinateur,
|
||||
NULL::text AS niv_val,
|
||||
NULL::text AS niveau_diffusion,
|
||||
n16.mnemonique AS floutage_dee,
|
||||
NULL::text AS methode_observation,
|
||||
CASE WHEN ts.name_source like 'MONITORING_%%'
|
||||
THEN split_part(ts.name_source,'_',2)
|
||||
ELSE 'RELEVE FLORE'::text
|
||||
END AS protocole,
|
||||
n6.mnemonique AS etat_biologique,
|
||||
n5.mnemonique AS statut_biologique,
|
||||
n10.mnemonique AS stade_vie,
|
||||
n19.mnemonique AS methode_determination,
|
||||
n3.mnemonique AS comportement,
|
||||
s.additional_data - '{effectif_textuel,strate_flore,code_atlas,id_nomenclature_braunblanq_abdom,id_nomenclature_strate_flore}'::text[] AS donnees_additionnelles
|
||||
FROM gn_synthese.synthese s
|
||||
JOIN ds ON ds.id_dataset = s.id_dataset
|
||||
JOIN af ON ds.id_acquisition_framework = af.id_acquisition_framework
|
||||
JOIN geo ON s.id_synthese = geo.id_synthese
|
||||
JOIN spatial_ref_sys sp ON st_srid(s.the_geom_4326) = sp.auth_srid
|
||||
LEFT JOIN ref_habitats.habref h ON h.cd_hab = s.cd_hab
|
||||
LEFT JOIN pr_occtax.t_releves_occtax occ ON occ.unique_id_sinp_grp = s.unique_id_sinp_grp
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n1 ON s.id_nomenclature_geo_object_nature = n1.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n2 ON s.id_nomenclature_grp_typ = n2.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n3 ON s.id_nomenclature_behaviour = n3.id_nomenclature
|
||||
-- LEFT JOIN ref_nomenclatures.t_nomenclatures n4 ON s.id_nomenclature_obs_technique = n4.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n5 ON s.id_nomenclature_bio_status = n5.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n6 ON s.id_nomenclature_bio_condition = n6.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n7 ON s.id_nomenclature_naturalness = n7.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n8 ON s.id_nomenclature_exist_proof = n8.id_nomenclature
|
||||
-- LEFT JOIN ref_nomenclatures.t_nomenclatures n9 ON s.id_nomenclature_diffusion_level = n9.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n10 ON s.id_nomenclature_life_stage = n10.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n11 ON s.id_nomenclature_sex = n11.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n12 ON s.id_nomenclature_obj_count = n12.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n13 ON s.id_nomenclature_type_count = n13.id_nomenclature
|
||||
-- LEFT JOIN ref_nomenclatures.t_nomenclatures n14 ON s.id_nomenclature_sensitivity = n14.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n15 ON s.id_nomenclature_observation_status = n15.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n16 ON s.id_nomenclature_blurring = n16.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n17 ON s.id_nomenclature_source_status = n17.id_nomenclature
|
||||
-- LEFT JOIN ref_nomenclatures.t_nomenclatures n18 ON s.id_nomenclature_info_geo_type = n18.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n19 ON s.id_nomenclature_determination_method = n19.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n20 ON s.id_nomenclature_valid_status = n20.id_nomenclature
|
||||
LEFT JOIN taxonomie.taxref n21 ON s.cd_nom = n21.cd_nom
|
||||
--JOIN gn_synthese.cor_area_synthese cas ON cas.id_synthese = s.id_synthese
|
||||
|
||||
-- ADD CEN_38
|
||||
JOIN gn_synthese.t_sources ts ON ts.id_source = s.id_source
|
||||
WHERE n21.regne::text = 'Plantae'::text
|
||||
-- exclision CA ([SICEN] Données anciennes,[TEST] : migration sicen,[SICEN] Hors Etude)
|
||||
AND ds.id_acquisition_framework NOT IN (53,66,1,2,6)
|
||||
-- exclision JDD `Observations opportunistes du CEN Isère importé depuis Faune Isère`
|
||||
AND ds.id_dataset NOT IN (185,377,236)
|
||||
-- exclision CA (`Gestion de l'Espace Naturel Sensible du Méandre des Oves`, `Gestion de la Réserve Naturelle Nationale de l'Ile de la Platière`, `Gestion des Natura 2000 FR 8201749 « Milieux alluviaux et aquatiques de l’Île de la Platière » et FR 8212012 « Île de la Platière »`, `RNN Platière`
|
||||
--AND (ts.id_module = 7 AND ds.id_acquisition_framework IN (7,38,39,42,44,45,46,48,55,57,58,60)
|
||||
AND (
|
||||
unaccent(observers) LIKE ANY (array['%%Conservatoire d%%Espaces Naturels Isere%%', '%%Conservatoire d%%Espaces Naturels d%%Isere%%', '%%CEN Isere%%', '%%Association des Amis de l%%Ile de la Platiere%%'])
|
||||
OR observers NOT LIKE '%%(%%)%%'
|
||||
)
|
||||
AND unaccent(observers) NOT LIKE ALL(array[
|
||||
'%%Sympetrum%%','%%Departement%%','%%Rhone%%','%%Oxalis%%',
|
||||
'%%LPO%%','%%GRPLS%%','%%Parvi%%','%%CD Isere%%','%%Personnel%%','%%Independant%%'
|
||||
'%%DELOCHE Denis%%'
|
||||
])
|
||||
AND observers NOT IN ('', 'Benoit Dodelin',', ')
|
||||
-- statut_validation = 'Certain - très probable'
|
||||
-- AND n20.cd_nomenclature = '1'
|
||||
ORDER BY s.id_synthese;
|
||||
'''
|
||||
|
||||
with con_gn.begin() as cnx:
|
||||
cnx.execute(sql)
|
||||
@@ -0,0 +1,172 @@
|
||||
DROP VIEW IF EXISTS gn_exports.v_synthese_sinp_with_metadata_for_gn2pg;
|
||||
|
||||
CREATE VIEW gn_exports.v_synthese_sinp_with_metadata_for_gn2pg AS
|
||||
WITH af_actors AS (SELECT cafa.id_acquisition_framework,
|
||||
json_build_object('type_role',
|
||||
CASE
|
||||
WHEN cafa.id_organism IS NOT NULL THEN 'organism'::TEXT
|
||||
WHEN cafa.id_role IS NOT NULL THEN 'role'::TEXT
|
||||
ELSE NULL::TEXT
|
||||
END, 'uuid_actor', coalesce(borg.uuid_organisme, tro.uuid_role),
|
||||
'cd_nomenclature_actor_role', tn.cd_nomenclature, 'identity',
|
||||
CASE
|
||||
WHEN cafa.id_organism IS NOT NULL
|
||||
THEN json_build_object('organism_name', borg.nom_organisme)
|
||||
WHEN cafa.id_role IS NOT NULL THEN json_build_object('first_name',
|
||||
tro.nom_role,
|
||||
'last_name',
|
||||
tro.prenom_role) END,
|
||||
'email', coalesce(borg.email_organisme, tro.email)) AS json_data
|
||||
FROM gn_meta.cor_acquisition_framework_actor cafa
|
||||
LEFT JOIN utilisateurs.bib_organismes borg ON cafa.id_organism = borg.id_organisme
|
||||
LEFT JOIN utilisateurs.t_roles tro ON cafa.id_role = tro.id_role
|
||||
JOIN ref_nomenclatures.t_nomenclatures tn
|
||||
ON cafa.id_nomenclature_actor_role = tn.id_nomenclature),
|
||||
af AS (SELECT taf.id_acquisition_framework,
|
||||
jsonb_build_object('uuid', taf.unique_acquisition_framework_id, 'name',
|
||||
taf.acquisition_framework_name,
|
||||
'desc', taf.acquisition_framework_desc, 'start_date',
|
||||
taf.acquisition_framework_start_date, 'end_date',
|
||||
taf.acquisition_framework_end_date,
|
||||
'initial_closing_date', taf.initial_closing_date, 'territorial_level',
|
||||
ntl.cd_nomenclature, 'financing_type', nft.cd_nomenclature, 'target_description',
|
||||
taf.target_description, 'ecologic_or_geologic_target',
|
||||
taf.ecologic_or_geologic_target, 'actors',
|
||||
json_agg(af_actors.json_data)) AS af_data
|
||||
FROM gn_meta.t_acquisition_frameworks taf
|
||||
JOIN af_actors ON af_actors.id_acquisition_framework = taf.id_acquisition_framework
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ntl
|
||||
ON taf.id_nomenclature_territorial_level = ntl.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures nft
|
||||
ON taf.id_nomenclature_financing_type = nft.id_nomenclature
|
||||
GROUP BY taf.id_acquisition_framework, taf.acquisition_framework_name, taf.acquisition_framework_desc,
|
||||
taf.acquisition_framework_start_date, taf.acquisition_framework_end_date, taf.initial_closing_date,
|
||||
ntl.cd_nomenclature, nft.cd_nomenclature),
|
||||
ds_actors AS (SELECT cda.id_dataset,
|
||||
json_build_object('type_role',
|
||||
CASE
|
||||
WHEN cda.id_organism IS NOT NULL THEN 'organism'::TEXT
|
||||
WHEN cda.id_role IS NOT NULL THEN 'role'::TEXT
|
||||
ELSE NULL::TEXT
|
||||
END, 'uuid_actor', coalesce(borg.uuid_organisme, tro.uuid_role),
|
||||
'cd_nomenclature_actor_role', tn.cd_nomenclature, 'identity',
|
||||
CASE
|
||||
WHEN cda.id_organism IS NOT NULL
|
||||
THEN json_build_object('organism_name', borg.nom_organisme)
|
||||
WHEN cda.id_role IS NOT NULL THEN json_build_object('first_name',
|
||||
tro.nom_role,
|
||||
'last_name',
|
||||
tro.prenom_role) END,
|
||||
'email', coalesce(borg.email_organisme, tro.email)) AS json_data
|
||||
FROM gn_meta.cor_dataset_actor cda
|
||||
LEFT JOIN utilisateurs.bib_organismes borg ON cda.id_organism = borg.id_organisme
|
||||
LEFT JOIN utilisateurs.t_roles tro ON cda.id_role = tro.id_role
|
||||
JOIN ref_nomenclatures.t_nomenclatures tn
|
||||
ON cda.id_nomenclature_actor_role = tn.id_nomenclature),
|
||||
ds AS (SELECT tds.id_dataset,
|
||||
tds.id_acquisition_framework,
|
||||
-- tds.additional_data,
|
||||
jsonb_build_object('uuid', tds.unique_dataset_id, 'name', tds.dataset_name, 'desc', tds.dataset_desc,
|
||||
'shortname', tds.dataset_shortname, 'data_type', ndt.cd_nomenclature,
|
||||
'collecting_method', ncm.cd_nomenclature, 'data_origin', ndo.cd_nomenclature,
|
||||
'dataset_objectif', ndso.cd_nomenclature, 'resource_type', nrt.cd_nomenclature,
|
||||
'source_status', nss.cd_nomenclature, 'territories', array_agg(DISTINCT
|
||||
ref_nomenclatures.get_cd_nomenclature(cdt.id_nomenclature_territory)),
|
||||
'actors', json_agg(ds_actors.json_data)) AS dataset_data
|
||||
FROM gn_meta.t_datasets tds
|
||||
JOIN ds_actors ON ds_actors.id_dataset = tds.id_dataset
|
||||
LEFT JOIN gn_meta.cor_dataset_territory cdt ON cdt.id_dataset = tds.id_dataset
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ndt
|
||||
ON tds.id_nomenclature_data_type = ndt.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ncm
|
||||
ON tds.id_nomenclature_collecting_method = ncm.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ndo
|
||||
ON tds.id_nomenclature_data_origin = ndo.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ndso
|
||||
ON tds.id_nomenclature_dataset_objectif = ndso.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures nrt
|
||||
ON tds.id_nomenclature_resource_type = nrt.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures nss
|
||||
ON tds.id_nomenclature_source_status = nss.id_nomenclature
|
||||
GROUP BY tds.id_dataset, tds.id_acquisition_framework, tds.unique_dataset_id, tds.dataset_name,
|
||||
tds.dataset_desc, tds.dataset_shortname, ndt.cd_nomenclature, ncm.cd_nomenclature,
|
||||
ndo.cd_nomenclature, ndso.cd_nomenclature, nrt.cd_nomenclature, nss.cd_nomenclature)
|
||||
SELECT s.id_synthese,
|
||||
s.entity_source_pk_value AS id_source,
|
||||
s.unique_id_sinp AS id_perm_sinp,
|
||||
s.unique_id_sinp_grp AS id_perm_grp_sinp,
|
||||
s.date_min AS date_debut,
|
||||
s.date_max AS date_fin,
|
||||
s.cd_nom,
|
||||
s.meta_v_taxref AS version_taxref,
|
||||
s.nom_cite,
|
||||
s.count_min AS nombre_min,
|
||||
s.count_max AS nombre_max,
|
||||
s.altitude_min,
|
||||
s.altitude_max,
|
||||
s.depth_min AS profondeur_min,
|
||||
s.depth_max AS profondeur_max,
|
||||
s.observers AS observateurs,
|
||||
s.determiner AS determinateur,
|
||||
s.validator AS validateur,
|
||||
s.sample_number_proof AS numero_preuve,
|
||||
s.digital_proof AS preuve_numerique,
|
||||
s.non_digital_proof AS preuve_non_numerique,
|
||||
s.comment_context AS comment_releve,
|
||||
s.comment_description AS comment_occurrence,
|
||||
ds.dataset_data AS jdd_data,
|
||||
af.af_data AS ca_data,
|
||||
s.reference_biblio,
|
||||
s.cd_hab AS code_habitat,
|
||||
h.lb_hab_fr AS habitat,
|
||||
s.place_name AS nom_lieu,
|
||||
s.precision,
|
||||
s.additional_data::TEXT AS donnees_additionnelles,
|
||||
st_astext(s.the_geom_4326) AS wkt_4326,
|
||||
n1.cd_nomenclature AS nature_objet_geo,
|
||||
n2.cd_nomenclature AS type_regroupement,
|
||||
s.grp_method AS methode_regroupement,
|
||||
n3.cd_nomenclature AS comportement,
|
||||
n4.cd_nomenclature AS technique_obs,
|
||||
n5.cd_nomenclature AS statut_biologique,
|
||||
n6.cd_nomenclature AS etat_biologique,
|
||||
n7.cd_nomenclature AS naturalite,
|
||||
n8.cd_nomenclature AS preuve_existante,
|
||||
n9.cd_nomenclature AS precision_diffusion,
|
||||
n10.cd_nomenclature AS stade_vie,
|
||||
n11.cd_nomenclature AS sexe,
|
||||
n12.cd_nomenclature AS objet_denombrement,
|
||||
n13.cd_nomenclature AS type_denombrement,
|
||||
n14.cd_nomenclature AS niveau_sensibilite,
|
||||
n15.cd_nomenclature AS statut_observation,
|
||||
n16.cd_nomenclature AS floutage_dee,
|
||||
n17.cd_nomenclature AS statut_source,
|
||||
n18.cd_nomenclature AS type_info_geo,
|
||||
n19.cd_nomenclature AS methode_determination,
|
||||
n20.cd_nomenclature AS statut_validation,
|
||||
coalesce(s.meta_update_date, s.meta_create_date) AS derniere_action
|
||||
FROM gn_synthese.synthese s
|
||||
JOIN ds ON ds.id_dataset = s.id_dataset
|
||||
JOIN af ON ds.id_acquisition_framework = af.id_acquisition_framework
|
||||
LEFT JOIN ref_habitats.habref h ON h.cd_hab = s.cd_hab
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n1 ON s.id_nomenclature_geo_object_nature = n1.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n2 ON s.id_nomenclature_grp_typ = n2.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n3 ON s.id_nomenclature_behaviour = n3.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n4 ON s.id_nomenclature_obs_technique = n4.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n5 ON s.id_nomenclature_bio_status = n5.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n6 ON s.id_nomenclature_bio_condition = n6.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n7 ON s.id_nomenclature_naturalness = n7.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n8 ON s.id_nomenclature_exist_proof = n8.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n9 ON s.id_nomenclature_diffusion_level = n9.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n10 ON s.id_nomenclature_life_stage = n10.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n11 ON s.id_nomenclature_sex = n11.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n12 ON s.id_nomenclature_obj_count = n12.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n13 ON s.id_nomenclature_type_count = n13.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n14 ON s.id_nomenclature_sensitivity = n14.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n15 ON s.id_nomenclature_observation_status = n15.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n16 ON s.id_nomenclature_blurring = n16.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n17 ON s.id_nomenclature_source_status = n17.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n18 ON s.id_nomenclature_info_geo_type = n18.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n19 ON s.id_nomenclature_determination_method = n19.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n20 ON s.id_nomenclature_valid_status = n20.id_nomenclature
|
||||
ORDER BY s.id_synthese;
|
||||
@@ -0,0 +1,52 @@
|
||||
import requests
|
||||
import geopandas as gpd
|
||||
|
||||
|
||||
def get_orb_token():
|
||||
login = 'cgeier'
|
||||
pwd = 'adm1n*cEn38'
|
||||
url = "https://donnees.biodiversite-auvergne-rhone-alpes.fr/api/auth/login"
|
||||
return requests.post(
|
||||
url,
|
||||
json={"login": login, "password": pwd},
|
||||
headers={'Content-Type': 'application/json'}
|
||||
)
|
||||
|
||||
|
||||
def get_orb_sp(tok, params: dict = None, geo=None):
|
||||
import json
|
||||
print("LOAD ORB DATA ...")
|
||||
url = "https://donnees.biodiversite-auvergne-rhone-alpes.fr/api/synthese/for_web?"
|
||||
if geo:
|
||||
params += {
|
||||
# "geoIntersection":{"type":"Feature","properties":{},"geometry":_json_geo['features'][0]['geometry']},
|
||||
"geoIntersection": geo.to_crs(4326).unary_union.wkt,
|
||||
"with_areas": False
|
||||
}
|
||||
|
||||
res = requests.post(
|
||||
url,
|
||||
headers={'Content-Type': 'application/json'},
|
||||
cookies=tok.cookies,
|
||||
json=params)
|
||||
|
||||
_orb_geo = gpd.read_file(res.text)
|
||||
orb_geo = (
|
||||
_orb_geo[_orb_geo.geom_type == 'Point']
|
||||
.reset_index(drop=True)
|
||||
.to_crs(2154))
|
||||
|
||||
orb_geo['cd_nom'] = [
|
||||
[cd['cd_nom'] for cd in json.loads(o)]
|
||||
for o in orb_geo.observations
|
||||
]
|
||||
orb = orb_geo.explode('cd_nom')
|
||||
# orb.drop(['observations','cd_nom'],axis=1,inplace=True)
|
||||
|
||||
return orb
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
tok = get_orb_token()
|
||||
orb = get_orb_sp(tok, params={'regne': 'Animalia'})
|
||||
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
|
||||
import geopandas as gpd
|
||||
import requests
|
||||
from os import path
|
||||
import io
|
||||
|
||||
|
||||
def get_zh(geom=None):
|
||||
url = 'https://geonature.cen-isere.fr/geonature/api/exports/api/9?limit=30000&{geo}'
|
||||
if not geom.empty :
|
||||
if geom.crs.srs.lower() != 'epsg:4326':
|
||||
geom.to_crs(4326,inplace=True)
|
||||
if gpd.__version__ >='1.0.1':
|
||||
geometry = 'geometry={geom}'.format(geom=geom.union_all().wkt)
|
||||
else:
|
||||
geometry = 'geometry={geom}'.format(geom=geom.unary_union.wkt)
|
||||
else:
|
||||
geom = ''
|
||||
print(url.format(geo=geometry))
|
||||
r = requests.get(url.format(geo=geometry))
|
||||
if r.status_code == 200:
|
||||
d = r.json()
|
||||
data = d['items']['features']
|
||||
p = [i['properties'] for i in data]
|
||||
g = [i['geometry'] for i in data]
|
||||
df = gpd.GeoDataFrame.from_features(data,crs=4326)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
PATH = '/media/cgeier/Disk2/tmp/GEONATURE/ST MARTIN URIAGE'
|
||||
FILE = 'ZH ENS POUR CEN 14102025.gpkg'
|
||||
f = gpd.read_file(path.join(PATH,FILE))
|
||||
df = get_zh(f)
|
||||
|
||||
# NEED GeoPandas >=1.0.1 pour sauvegarder
|
||||
df.to_file(path.join(PATH,'inventaire_zoneshumides_seglieres.gpkg'))
|
||||
@@ -0,0 +1,276 @@
|
||||
from pycen import con_gn
|
||||
|
||||
sql = '''
|
||||
DROP VIEW IF EXISTS gn_exports.v_synthese_sinp_with_metadata_invertebre_for_gn2pg;
|
||||
|
||||
CREATE VIEW gn_exports.v_synthese_sinp_with_metadata_invertebre_for_gn2pg AS
|
||||
WITH af_actors AS (SELECT cafa.id_acquisition_framework,
|
||||
json_build_object('type_role',
|
||||
CASE
|
||||
WHEN cafa.id_organism IS NOT NULL THEN 'organism'::TEXT
|
||||
WHEN cafa.id_role IS NOT NULL THEN 'role'::TEXT
|
||||
ELSE NULL::TEXT
|
||||
END, 'uuid_actor', coalesce(borg.uuid_organisme, tro.uuid_role),
|
||||
'cd_nomenclature_actor_role', tn.cd_nomenclature, 'identity',
|
||||
CASE
|
||||
WHEN cafa.id_organism IS NOT NULL
|
||||
THEN json_build_object('organism_name', borg.nom_organisme)
|
||||
WHEN cafa.id_role IS NOT NULL THEN json_build_object('first_name',
|
||||
tro.nom_role,
|
||||
'last_name',
|
||||
tro.prenom_role) END,
|
||||
'email', coalesce(borg.email_organisme, tro.email)) AS json_data
|
||||
FROM gn_meta.cor_acquisition_framework_actor cafa
|
||||
LEFT JOIN utilisateurs.bib_organismes borg ON cafa.id_organism = borg.id_organisme
|
||||
LEFT JOIN utilisateurs.t_roles tro ON cafa.id_role = tro.id_role
|
||||
JOIN ref_nomenclatures.t_nomenclatures tn
|
||||
ON cafa.id_nomenclature_actor_role = tn.id_nomenclature),
|
||||
af_territories AS (SELECT caft.id_acquisition_framework,
|
||||
array_agg(DISTINCT t_nomenclatures.cd_nomenclature) AS territories
|
||||
FROM gn_meta.cor_acquisition_framework_territory caft
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures
|
||||
ON caft.id_nomenclature_territory =
|
||||
t_nomenclatures.id_nomenclature
|
||||
GROUP BY caft.id_acquisition_framework),
|
||||
af_objectives AS (SELECT cafo.id_acquisition_framework,
|
||||
array_agg(DISTINCT t_nomenclatures.cd_nomenclature) AS objectives
|
||||
FROM gn_meta.cor_acquisition_framework_objectif cafo
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures
|
||||
ON cafo.id_nomenclature_objectif =
|
||||
t_nomenclatures.id_nomenclature
|
||||
GROUP BY cafo.id_acquisition_framework),
|
||||
af_voletsinp AS (SELECT cafv.id_acquisition_framework,
|
||||
array_agg(DISTINCT t_nomenclatures.cd_nomenclature) AS voletsinp
|
||||
FROM gn_meta.cor_acquisition_framework_voletsinp cafv
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures
|
||||
ON cafv.id_nomenclature_voletsinp =
|
||||
t_nomenclatures.id_nomenclature
|
||||
GROUP BY cafv.id_acquisition_framework),
|
||||
af_publication AS (SELECT cafp.id_acquisition_framework,
|
||||
array_agg(DISTINCT
|
||||
jsonb_build_object('uuid', sinp_datatype_publications.unique_publication_id,
|
||||
'reference',
|
||||
sinp_datatype_publications.publication_reference,
|
||||
'url',
|
||||
sinp_datatype_publications.publication_url)) AS publications
|
||||
FROM gn_meta.cor_acquisition_framework_publication cafp
|
||||
LEFT JOIN gn_meta.sinp_datatype_publications
|
||||
ON cafp.id_publication = sinp_datatype_publications.id_publication
|
||||
GROUP BY cafp.id_acquisition_framework)
|
||||
,
|
||||
af AS
|
||||
(SELECT taf.id_acquisition_framework,
|
||||
jsonb_build_object('uuid', taf.unique_acquisition_framework_id, 'name',
|
||||
taf.acquisition_framework_name,
|
||||
'desc', taf.acquisition_framework_desc, 'start_date',
|
||||
taf.acquisition_framework_start_date, 'end_date',
|
||||
taf.acquisition_framework_end_date,
|
||||
'initial_closing_date', taf.initial_closing_date, 'territories',
|
||||
af_territories.territories, 'territorial_level',
|
||||
ntl.cd_nomenclature, 'territory_desc', taf.territory_desc, 'objectives',
|
||||
af_objectives.objectives, 'publications', af_publication.publications,
|
||||
'financing_type', nft.cd_nomenclature,
|
||||
'target_description',
|
||||
taf.target_description, 'ecologic_or_geologic_target',
|
||||
taf.ecologic_or_geologic_target, 'actors',
|
||||
json_agg(af_actors.json_data), 'is_parent', taf.is_parent, 'parent_uuid',
|
||||
tafp.unique_acquisition_framework_id) AS af_data
|
||||
FROM gn_meta.t_acquisition_frameworks taf
|
||||
LEFT JOIN gn_meta.t_acquisition_frameworks tafp
|
||||
ON tafp.id_acquisition_framework = taf.acquisition_framework_parent_id
|
||||
JOIN af_actors ON af_actors.id_acquisition_framework = taf.id_acquisition_framework
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ntl
|
||||
ON taf.id_nomenclature_territorial_level = ntl.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures nft
|
||||
ON taf.id_nomenclature_financing_type = nft.id_nomenclature
|
||||
LEFT JOIN af_territories ON af_territories.id_acquisition_framework = taf.id_acquisition_framework
|
||||
LEFT JOIN af_objectives ON af_objectives.id_acquisition_framework = taf.id_acquisition_framework
|
||||
LEFT JOIN af_voletsinp ON af_voletsinp.id_acquisition_framework = taf.id_acquisition_framework
|
||||
LEFT JOIN af_publication ON af_publication.id_acquisition_framework = taf.id_acquisition_framework
|
||||
GROUP BY taf.id_acquisition_framework, taf.acquisition_framework_name, taf.acquisition_framework_desc,
|
||||
taf.acquisition_framework_start_date, taf.acquisition_framework_end_date, taf.initial_closing_date,
|
||||
ntl.cd_nomenclature, nft.cd_nomenclature, af_territories.territories, af_objectives.objectives,
|
||||
af_voletsinp.voletsinp, af_publication.publications, taf.is_parent,
|
||||
tafp.unique_acquisition_framework_id),
|
||||
ds_actors AS (SELECT cda.id_dataset,
|
||||
json_build_object('type_role',
|
||||
CASE
|
||||
WHEN cda.id_organism IS NOT NULL THEN 'organism'::TEXT
|
||||
WHEN cda.id_role IS NOT NULL THEN 'role'::TEXT
|
||||
ELSE NULL::TEXT
|
||||
END, 'uuid_actor', coalesce(borg.uuid_organisme, tro.uuid_role),
|
||||
'cd_nomenclature_actor_role', tn.cd_nomenclature, 'identity',
|
||||
CASE
|
||||
WHEN cda.id_organism IS NOT NULL
|
||||
THEN json_build_object('organism_name', borg.nom_organisme)
|
||||
WHEN cda.id_role IS NOT NULL THEN json_build_object('first_name',
|
||||
tro.nom_role,
|
||||
'last_name',
|
||||
tro.prenom_role) END,
|
||||
'email', coalesce(borg.email_organisme, tro.email)) AS json_data
|
||||
FROM gn_meta.cor_dataset_actor cda
|
||||
LEFT JOIN utilisateurs.bib_organismes borg ON cda.id_organism = borg.id_organisme
|
||||
LEFT JOIN utilisateurs.t_roles tro ON cda.id_role = tro.id_role
|
||||
JOIN ref_nomenclatures.t_nomenclatures tn
|
||||
ON cda.id_nomenclature_actor_role = tn.id_nomenclature),
|
||||
ds_protocols AS (SELECT cdp.id_dataset,
|
||||
jsonb_build_object(
|
||||
'uuid', sdp.unique_protocol_id,
|
||||
'name', sdp.protocol_name,
|
||||
'desc', sdp.protocol_desc,
|
||||
'url', sdp.protocol_url,
|
||||
'type', t_nomenclatures.cd_nomenclature) AS protocols
|
||||
FROM gn_meta.cor_dataset_protocol cdp
|
||||
JOIN gn_meta.sinp_datatype_protocols sdp
|
||||
ON cdp.id_protocol = sdp.id_protocol
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures
|
||||
ON sdp.id_nomenclature_protocol_type = t_nomenclatures.id_nomenclature)
|
||||
,
|
||||
ds AS (SELECT tds.id_dataset,
|
||||
tds.id_acquisition_framework,
|
||||
-- tds.additional_data,
|
||||
jsonb_build_object('uuid', tds.unique_dataset_id, 'name', tds.dataset_name, 'desc', tds.dataset_desc,
|
||||
'shortname', tds.dataset_shortname, 'data_type', ndt.cd_nomenclature,
|
||||
'keywords', tds.keywords, 'marine_domain', tds.marine_domain,
|
||||
'terrestrial_domain', tds.terrestrial_domain, 'collecting_method',
|
||||
ncm.cd_nomenclature, 'protocols',
|
||||
ds_protocols.protocols,
|
||||
'data_origin', ndo.cd_nomenclature,
|
||||
'dataset_objectif', ndso.cd_nomenclature, 'resource_type', nrt.cd_nomenclature,
|
||||
'source_status', nss.cd_nomenclature, 'territories', array_agg(DISTINCT
|
||||
ref_nomenclatures.get_cd_nomenclature(cdt.id_nomenclature_territory)),
|
||||
'actors', json_agg(ds_actors.json_data)) AS dataset_data
|
||||
FROM gn_meta.t_datasets tds
|
||||
JOIN ds_actors ON ds_actors.id_dataset = tds.id_dataset
|
||||
LEFT JOIN gn_meta.cor_dataset_territory cdt ON cdt.id_dataset = tds.id_dataset
|
||||
LEFT JOIN ds_protocols ON ds_protocols.id_dataset = tds.id_dataset
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ndt
|
||||
ON tds.id_nomenclature_data_type = ndt.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ncm
|
||||
ON tds.id_nomenclature_collecting_method = ncm.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ndo
|
||||
ON tds.id_nomenclature_data_origin = ndo.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ndso
|
||||
ON tds.id_nomenclature_dataset_objectif = ndso.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures nrt
|
||||
ON tds.id_nomenclature_resource_type = nrt.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures nss
|
||||
ON tds.id_nomenclature_source_status = nss.id_nomenclature
|
||||
GROUP BY tds.id_dataset, tds.id_acquisition_framework, tds.unique_dataset_id, tds.dataset_name,
|
||||
tds.dataset_desc, tds.dataset_shortname, ndt.cd_nomenclature, ncm.cd_nomenclature,
|
||||
ndo.cd_nomenclature, ndso.cd_nomenclature, nrt.cd_nomenclature, nss.cd_nomenclature,
|
||||
ds_protocols.protocols)
|
||||
SELECT s.id_synthese,
|
||||
s.entity_source_pk_value AS id_source,
|
||||
s.unique_id_sinp AS id_perm_sinp,
|
||||
s.unique_id_sinp_grp AS id_perm_grp_sinp,
|
||||
s.date_min AS date_debut,
|
||||
s.date_max AS date_fin,
|
||||
s.cd_nom,
|
||||
s.meta_v_taxref AS version_taxref,
|
||||
s.nom_cite,
|
||||
s.count_min AS nombre_min,
|
||||
s.count_max AS nombre_max,
|
||||
s.altitude_min,
|
||||
s.altitude_max,
|
||||
s.depth_min AS profondeur_min,
|
||||
s.depth_max AS profondeur_max,
|
||||
s.observers AS observateurs,
|
||||
s.determiner AS determinateur,
|
||||
s.validator AS validateur,
|
||||
s.sample_number_proof AS numero_preuve,
|
||||
s.digital_proof AS preuve_numerique,
|
||||
s.non_digital_proof AS preuve_non_numerique,
|
||||
s.comment_context AS comment_releve,
|
||||
s.comment_description AS comment_occurrence,
|
||||
ds.dataset_data AS jdd_data,
|
||||
af.af_data AS ca_data,
|
||||
s.reference_biblio,
|
||||
s.cd_hab AS code_habitat,
|
||||
h.lb_hab_fr AS habitat,
|
||||
s.place_name AS nom_lieu,
|
||||
s.precision,
|
||||
s.additional_data AS donnees_additionnelles,
|
||||
st_astext(s.the_geom_4326) AS wkt_4326,
|
||||
n1.cd_nomenclature AS nature_objet_geo,
|
||||
n2.cd_nomenclature AS type_regroupement,
|
||||
s.grp_method AS methode_regroupement,
|
||||
n3.cd_nomenclature AS comportement,
|
||||
n4.cd_nomenclature AS technique_obs,
|
||||
n5.cd_nomenclature AS statut_biologique,
|
||||
n6.cd_nomenclature AS etat_biologique,
|
||||
n7.cd_nomenclature AS naturalite,
|
||||
n8.cd_nomenclature AS preuve_existante,
|
||||
n9.cd_nomenclature AS precision_diffusion,
|
||||
n10.cd_nomenclature AS stade_vie,
|
||||
n11.cd_nomenclature AS sexe,
|
||||
n12.cd_nomenclature AS objet_denombrement,
|
||||
n13.cd_nomenclature AS type_denombrement,
|
||||
n14.cd_nomenclature AS niveau_sensibilite,
|
||||
n15.cd_nomenclature AS statut_observation,
|
||||
n16.cd_nomenclature AS floutage_dee,
|
||||
n17.cd_nomenclature AS statut_source,
|
||||
n18.cd_nomenclature AS type_info_geo,
|
||||
n19.cd_nomenclature AS methode_determination,
|
||||
n20.cd_nomenclature AS statut_validation,
|
||||
coalesce(s.meta_update_date, s.meta_create_date) AS derniere_action
|
||||
FROM gn_synthese.synthese s
|
||||
JOIN ds ON ds.id_dataset = s.id_dataset
|
||||
JOIN af ON ds.id_acquisition_framework = af.id_acquisition_framework
|
||||
LEFT JOIN ref_habitats.habref h ON h.cd_hab = s.cd_hab
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n1 ON s.id_nomenclature_geo_object_nature = n1.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n2 ON s.id_nomenclature_grp_typ = n2.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n3 ON s.id_nomenclature_behaviour = n3.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n4 ON s.id_nomenclature_obs_technique = n4.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n5 ON s.id_nomenclature_bio_status = n5.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n6 ON s.id_nomenclature_bio_condition = n6.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n7 ON s.id_nomenclature_naturalness = n7.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n8 ON s.id_nomenclature_exist_proof = n8.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n9 ON s.id_nomenclature_diffusion_level = n9.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n10 ON s.id_nomenclature_life_stage = n10.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n11 ON s.id_nomenclature_sex = n11.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n12 ON s.id_nomenclature_obj_count = n12.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n13 ON s.id_nomenclature_type_count = n13.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n14 ON s.id_nomenclature_sensitivity = n14.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n15 ON s.id_nomenclature_observation_status = n15.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n16 ON s.id_nomenclature_blurring = n16.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n17 ON s.id_nomenclature_source_status = n17.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n18 ON s.id_nomenclature_info_geo_type = n18.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n19 ON s.id_nomenclature_determination_method = n19.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n20 ON s.id_nomenclature_valid_status = n20.id_nomenclature
|
||||
|
||||
-- ADD CEN_38
|
||||
JOIN taxonomie.taxref t ON t.cd_nom = s.cd_nom
|
||||
JOIN gn_synthese.t_sources ts ON ts.id_source = s.id_source
|
||||
|
||||
WHERE t.regne = 'Animalia'
|
||||
AND t.phylum <> 'Chordata'
|
||||
-- exclision CA ([SICEN] Données anciennes,[TEST] : migration sicen,[SICEN] Hors Etude)
|
||||
AND ds.id_acquisition_framework NOT IN (
|
||||
53,1,2,6, -- CA tests
|
||||
84, -- Echange de données PG Chravas
|
||||
85, -- Echange de données
|
||||
116 -- Echange de données issus de SICEN
|
||||
)
|
||||
-- exclision CA (`Gestion de l'Espace Naturel Sensible du Méandre des Oves`, `Gestion de la Réserve Naturelle Nationale de l'Ile de la Platière`, `Gestion des Natura 2000 FR 8201749 « Milieux alluviaux et aquatiques de l’Île de la Platière » et FR 8212012 « Île de la Platière »`, `RNN Platière`
|
||||
--AND (ts.id_module = 7 AND ds.id_acquisition_framework IN (7,38,39,42,44,45,46,48,55,57,58,60)
|
||||
AND (
|
||||
unaccent(observers) LIKE ANY (array['%%Conservatoire d%%Espaces Naturels Isere%%', '%%Conservatoire d%%Espaces Naturels d%%Isere%%', '%%CEN Isere%%', '%%Association des Amis de l%%Ile de la Platiere%%'])
|
||||
OR observers NOT LIKE '%%(%%)%%'
|
||||
)
|
||||
AND unaccent(observers) NOT LIKE ALL(array[
|
||||
'%%Sympetrum%%','%%Departement%%','%%Rhone%%','%%Oxalis%%',
|
||||
'%%LPO%%','%%GRPLS%%','%%Parvi%%','%%CD Isere%%','%%CBNA%%',
|
||||
'%%Flavia%%','%%Gentiana%%','%%region%%','%%Personnel%%','%%Independant%%'
|
||||
])
|
||||
AND observers NOT IN ('', 'Benoit Dodelin',', ')
|
||||
AND ds.id_dataset NOT IN (185,377,236,409)
|
||||
-- exclision JDD `Observations opportunistes du CEN Isère importé depuis Faune Isère`
|
||||
-- statut_validation = 'Certain - très probable'
|
||||
-- AND n20.cd_nomenclature = '1'
|
||||
AND s.cd_nom != 781926 -- Paradiarsia punicea (Hübner, 1803) ; Espèce en danger critique à faire remonter une fois la liste de sensibilité invertébrée sortie !
|
||||
ORDER BY s.id_synthese;
|
||||
'''
|
||||
|
||||
with con_gn.begin() as cnx:
|
||||
cnx.execute(sql)
|
||||
@@ -0,0 +1,75 @@
|
||||
from pycen import con_gn
|
||||
|
||||
sql = """
|
||||
DROP VIEW IF EXISTS gn_exports.v_synthese_sinp_acquisition_flore_with_metadata_for_gn2pg CASCADE;
|
||||
|
||||
CREATE OR REPLACE VIEW gn_exports.v_synthese_sinp_acquisition_flore_with_metadata_for_gn2pg AS
|
||||
WITH af_actors AS
|
||||
(SELECT cafa.*
|
||||
FROM gn_meta.cor_acquisition_framework_actor cafa
|
||||
LEFT JOIN utilisateurs.bib_organismes borg ON cafa.id_organism = borg.id_organisme
|
||||
--WHERE borg.uuid_organisme = '5a433bd0-2070-25d9-e053-2614a8c026f8'::uuid
|
||||
--OR borg.uuid_organisme = 'a3c44538-f22a-4134-9e21-28646a73986b'::uuid
|
||||
), act AS
|
||||
(SELECT *
|
||||
FROM crosstab( 'SELECT id_acquisition_framework, id_nomenclature_actor_role, id_organism
|
||||
FROM gn_meta.cor_acquisition_framework_actor
|
||||
WHERE id_acquisition_framework NOT IN (1,6)
|
||||
ORDER BY 1,2' , $$VALUES ('359'::text), ('361')$$ ) AS ct ("id_acquisition_framework" int, "Contact principal" int, "Producteur du jeu de données" int))
|
||||
SELECT taf.id_acquisition_framework AS id_ca,
|
||||
taf.unique_acquisition_framework_id AS id_sinp_ca,
|
||||
taf.acquisition_framework_name AS libelle,
|
||||
NULL::text AS cadre_acquisition_parent,
|
||||
taf.acquisition_framework_desc AS description,
|
||||
taf.keywords AS mots_cles,
|
||||
ref1.label_default as niveau_territorial,
|
||||
taf.territory_desc AS description_territoire,
|
||||
json_agg(ref2.label_default) AS objectifs_cadre_acquisition,
|
||||
ref4.label_default AS volet_SINP,
|
||||
ref3.label_default AS type_financement,
|
||||
taf.target_description AS description_cible,
|
||||
to_char(taf.meta_create_date, 'DD-MM-YYYY')::date AS date_debut,
|
||||
NULL::date AS date_fin,
|
||||
ref5.nom_organisme AS acteur1_organisme,
|
||||
NULL::text AS acteur1_personne_groupe,
|
||||
'Contact principal' AS acteur1_type_role,
|
||||
ref6.nom_organisme AS acteur2_organisme,
|
||||
NULL::text AS acteur2_personne_groupe,
|
||||
CASE
|
||||
WHEN ref6.nom_organisme IS NOT NULL THEN 'Producteur du jeu de données'
|
||||
ELSE NULL
|
||||
END AS acteur2_type_role,
|
||||
NULL::text AS acteur3_organisme,
|
||||
NULL::text AS acteur3_personne_groupe,
|
||||
NULL::text AS acteur3_type_role
|
||||
FROM gn_meta.t_acquisition_frameworks taf
|
||||
left JOIN af_actors ON af_actors.id_acquisition_framework = taf.id_acquisition_framework
|
||||
left JOIN gn_meta.cor_acquisition_framework_objectif obj ON obj.id_acquisition_framework = taf.id_acquisition_framework
|
||||
left JOIN ref_nomenclatures.t_nomenclatures ref1 ON ref1.id_nomenclature = taf.id_nomenclature_territorial_level
|
||||
JOIN ref_nomenclatures.t_nomenclatures ref2 ON ref2.id_nomenclature = obj.id_nomenclature_objectif
|
||||
JOIN ref_nomenclatures.t_nomenclatures ref3 ON ref3.id_nomenclature = taf.id_nomenclature_financing_type
|
||||
left JOIN gn_meta.cor_acquisition_framework_voletsinp sinp ON sinp.id_acquisition_framework = taf.id_acquisition_framework
|
||||
left JOIN ref_nomenclatures.t_nomenclatures ref4 ON ref4.id_nomenclature = sinp.id_nomenclature_voletsinp
|
||||
left JOIN act ON act.id_acquisition_framework = taf.id_acquisition_framework
|
||||
LEFT JOIN utilisateurs.bib_organismes ref5 ON ref5.id_organisme = act."Contact principal"
|
||||
LEFT JOIN utilisateurs.bib_organismes ref6 ON ref6.id_organisme = act."Producteur du jeu de données"
|
||||
join gn_exports.v_synthese_sinp_with_metadata_flora_for_gn2pg vsswmffgp on vsswmffgp.id_sinp_ca = taf.unique_acquisition_framework_id
|
||||
GROUP BY taf.id_acquisition_framework,
|
||||
taf.unique_acquisition_framework_id,
|
||||
taf.acquisition_framework_name,
|
||||
taf.acquisition_framework_desc,
|
||||
taf.keywords,
|
||||
ref1.label_default,
|
||||
taf.territory_desc,
|
||||
ref4.label_default,
|
||||
ref3.label_default,
|
||||
taf.target_description,
|
||||
to_char(taf.meta_create_date, 'DD-MM-YYYY')::date,
|
||||
ref5.nom_organisme,
|
||||
ref6.nom_organisme
|
||||
order by 1
|
||||
;
|
||||
"""
|
||||
|
||||
with con_gn.begin() as cnx:
|
||||
cnx.execute(sql)
|
||||
@@ -0,0 +1,87 @@
|
||||
from pycen import con_gn
|
||||
|
||||
sql = '''
|
||||
DROP VIEW IF EXISTS gn_exports.v_synthese_sinp_jeu_donnees_flore_with_metadata_for_gn2pg CASCADE;
|
||||
|
||||
CREATE VIEW gn_exports.v_synthese_sinp_jeu_donnees_flore_with_metadata_for_gn2pg AS
|
||||
WITH ds_actors AS (
|
||||
SELECT cda.id_dataset,
|
||||
borg.nom_organisme,
|
||||
cda.id_role,
|
||||
borg.uuid_organisme,
|
||||
tro.nom_role,
|
||||
tro.prenom_role,
|
||||
borg.email_organisme,
|
||||
tro.email
|
||||
FROM gn_meta.cor_dataset_actor cda
|
||||
LEFT JOIN utilisateurs.bib_organismes borg ON cda.id_organism = borg.id_organisme
|
||||
LEFT JOIN utilisateurs.t_roles tro ON cda.id_role = tro.id_role
|
||||
JOIN ref_nomenclatures.t_nomenclatures tn ON cda.id_nomenclature_actor_role = tn.id_nomenclature
|
||||
-- WHERE borg.uuid_organisme = '5a433bd0-2070-25d9-e053-2614a8c026f8'::uuid
|
||||
-- OR borg.uuid_organisme = 'a3c44538-f22a-4134-9e21-28646a73986b'::uuid
|
||||
),
|
||||
actor AS (
|
||||
SELECT *
|
||||
FROM crosstab( 'SELECT id_dataset, id_nomenclature_actor_role, id_organism
|
||||
FROM gn_meta.cor_dataset_actor
|
||||
ORDER BY 1,2' ,
|
||||
$$VALUES ('359'), ('364'), ('365'), ('395')$$ ) AS ct ("id_dataset" int, "Contact principal" int, "Producteur du jeu de données" int, "Point de contact base de données de production" int, "Point de contact pour les métadonnées" int)
|
||||
),
|
||||
ds AS (
|
||||
SELECT distinct tds.id_dataset AS id_jdd,
|
||||
tds.unique_dataset_id AS id_sinp_jdd,
|
||||
acq.acquisition_framework_name AS cadre_acquisiton,
|
||||
tds.dataset_name AS nom_jeu_donnees,
|
||||
tds.dataset_shortname AS nom_court,
|
||||
tds.dataset_desc AS description,
|
||||
ndso.label_default AS objectif,
|
||||
ncm.label_default AS methode_collecte,
|
||||
ndo.label_default AS origine_donnees,
|
||||
nss.label_default AS statut_source,
|
||||
tds.keywords AS mots_cles,
|
||||
act1.nom_organisme AS acteur1_organisme,
|
||||
NULL::text AS acteur1_personne_groupe,
|
||||
'Contact principal' AS acteur1_type_role,
|
||||
act2.nom_organisme AS acteur2_organisme,
|
||||
NULL::text AS acteur2_personne_groupe,
|
||||
CASE
|
||||
WHEN act2.nom_organisme IS NOT NULL THEN 'Producteur du jeu de données'
|
||||
ELSE NULL
|
||||
END AS acteur2_type_role,
|
||||
act3.nom_organisme AS acteur3_organisme,
|
||||
NULL::text AS acteur3_personne_groupe,
|
||||
CASE
|
||||
WHEN act3.nom_organisme IS NOT NULL THEN 'Point de contact base de données de production'
|
||||
ELSE NULL
|
||||
END AS acteur3_type_role,
|
||||
act4.nom_organisme AS acteur4_organisme,
|
||||
NULL::text AS acteur4_personne_groupe,
|
||||
CASE
|
||||
WHEN act4.nom_organisme IS NOT NULL THEN 'Point de contact pour les métadonnées'
|
||||
ELSE NULL
|
||||
END AS acteur4_type_role
|
||||
FROM gn_meta.t_datasets tds
|
||||
left JOIN ds_actors ON ds_actors.id_dataset = tds.id_dataset
|
||||
left JOIN actor ON actor.id_dataset = tds.id_dataset
|
||||
JOIN gn_meta.t_acquisition_frameworks acq ON tds.id_acquisition_framework = acq.id_acquisition_framework
|
||||
LEFT JOIN gn_meta.cor_dataset_territory cdt ON cdt.id_dataset = tds.id_dataset
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ndt ON tds.id_nomenclature_data_type = ndt.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ncm ON tds.id_nomenclature_collecting_method = ncm.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ndo ON tds.id_nomenclature_data_origin = ndo.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ndso ON tds.id_nomenclature_dataset_objectif = ndso.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures nrt ON tds.id_nomenclature_resource_type = nrt.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures nss ON tds.id_nomenclature_source_status = nss.id_nomenclature
|
||||
LEFT JOIN utilisateurs.bib_organismes act1 ON actor."Contact principal" = act1.id_organisme
|
||||
LEFT JOIN utilisateurs.bib_organismes act2 ON actor."Producteur du jeu de données" = act2.id_organisme
|
||||
LEFT JOIN utilisateurs.bib_organismes act3 ON actor."Point de contact base de données de production" = act3.id_organisme
|
||||
LEFT JOIN utilisateurs.bib_organismes act4 ON actor."Point de contact pour les métadonnées" = act4.id_organisme
|
||||
join gn_exports.v_synthese_sinp_with_metadata_flora_for_gn2pg vsswmffgp on vsswmffgp.id_sinp_jdd = tds.unique_dataset_id
|
||||
GROUP BY tds.id_dataset, acq.acquisition_framework_name, tds.unique_dataset_id, tds.dataset_name, tds.dataset_shortname, tds.dataset_desc, ndso.label_default, ncm.label_default, ndo.label_default, nss.label_default, ds_actors.nom_organisme, act1.nom_organisme, act2.nom_organisme, act3.nom_organisme, act4.nom_organisme
|
||||
)
|
||||
SELECT * FROM ds order by 1
|
||||
--WHERE acteur1_organisme IN ('Conservatoire d’espaces naturels Isère')
|
||||
;
|
||||
'''
|
||||
|
||||
with con_gn.begin() as cnx:
|
||||
cnx.execute(sql)
|
||||
@@ -0,0 +1,20 @@
|
||||
from pycen import con_gn
|
||||
|
||||
sql = '''
|
||||
CREATE OR REPLACE VIEW gn_exports.v_bam_widget AS
|
||||
SELECT
|
||||
t.cd_ref AS cd_ref,
|
||||
t.nom_vern AS nom_vernaculaire,
|
||||
t.lb_nom AS nom_scientifique,
|
||||
s.id_synthese AS id_synthese,
|
||||
s.date_min AS date_min,
|
||||
s.date_max AS date_max,
|
||||
s.the_geom_4326 AS the_geom_4326,
|
||||
t.classe AS classe
|
||||
FROM gn_synthese.synthese s
|
||||
JOIN taxonomie.taxref t USING(cd_nom)
|
||||
;
|
||||
'''
|
||||
|
||||
with con_gn.begin() as cnx:
|
||||
cnx.execute(sql)
|
||||
@@ -0,0 +1,41 @@
|
||||
from pycen import con_gn
|
||||
|
||||
sql = '''
|
||||
DROP VIEW IF EXISTS gn_exports.v_eee_n2000_platiere;
|
||||
|
||||
CREATE OR REPLACE VIEW gn_exports.v_eee_n2000_platiere
|
||||
AS
|
||||
SELECT
|
||||
v.*
|
||||
FROM gn_synthese.v_synthese_for_export v
|
||||
JOIN ref_geo.l_areas ON st_intersects(ST_GeomFromGeoJSON(v.geojson_4326),st_transform(l_areas.geom,4326))
|
||||
JOIN ref_geo.bib_areas_types bib USING (id_type)
|
||||
WHERE bib.type_code = 'ZPS'
|
||||
AND area_name ilike '%%plati%%re%%'
|
||||
AND v.cd_ref in (100330, 101055, 101237, 101286, 103139, 103543, 103545, 103547,
|
||||
103557, 104074, 104805, 105017, 105433, 105615, 105689, 105960,
|
||||
106252, 106571, 106742, 106748, 106754, 106965, 107446, 108628,
|
||||
108642, 108810, 109141, 109911, 109926, 109954, 109980, 110762,
|
||||
111863, 111881, 112100, 112111, 112130, 112195, 112463, 112467,
|
||||
112482, 112712, 112790, 113418, 114024, 115527, 116089, 116137,
|
||||
116485, 117503, 117505, 117723, 117860, 117937, 118477, 119474,
|
||||
119558, 119595, 119854, 122630, 124025, 124164, 124168, 124378,
|
||||
124635, 124646, 124719, 124730, 125331, 128504, 128748, 128863,
|
||||
128954, 129468, 130484, 130491, 159690, 159937, 4419, 446169,
|
||||
448413, 454932, 4872, 611690, 611753, 612522, 717136, 717180,
|
||||
79766, 79877, 80086, 80824, 81955, 81978, 82018, 82080,
|
||||
82093, 82164, 83938, 84057, 84251, 85186, 85469, 85949,
|
||||
85957, 86167, 86513, 86817, 86869, 86975, 89452, 90192,
|
||||
90234, 92572, 92649, 92663, 92793, 93020, 93129, 93613,
|
||||
93923, 93924, 94168, 94489, 95679, 95823, 95831, 95965,
|
||||
95975, 95980, 96149, 96585, 96644, 96739, 96749, 96775,
|
||||
97346, 97594, 97666, 97961, 99260, 99359, 95983, 96814,
|
||||
117507, 129959, 81992, 90532, 97623, 104353, 111886, 112483,
|
||||
125369, 109037, 96624, 101056, 106800, 109949, 116762, 125324
|
||||
)
|
||||
;
|
||||
'''
|
||||
|
||||
with con_gn.begin() as cnx:
|
||||
cnx.execute(sql)
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
from pycen import con_gn
|
||||
|
||||
sql = '''
|
||||
|
||||
DROP VIEW IF EXISTS gn_exports.v_synthese_negria_forcenra;
|
||||
|
||||
CREATE OR REPLACE VIEW gn_exports.v_synthese_negria_forcenra
|
||||
AS
|
||||
WITH af_actors AS (
|
||||
SELECT
|
||||
cafa.id_acquisition_framework,
|
||||
json_build_object('type_role',
|
||||
CASE
|
||||
WHEN cafa.id_organism IS NOT NULL THEN 'organism'::TEXT
|
||||
WHEN cafa.id_role IS NOT NULL THEN 'role'::TEXT
|
||||
ELSE NULL::TEXT
|
||||
END, 'uuid_actor', coalesce(borg.uuid_organisme, tro.uuid_role),
|
||||
'cd_nomenclature_actor_role', tn.cd_nomenclature, 'identity',
|
||||
CASE
|
||||
WHEN cafa.id_organism IS NOT NULL
|
||||
THEN json_build_object('organism_name', borg.nom_organisme)
|
||||
WHEN cafa.id_role IS NOT NULL THEN json_build_object('first_name',
|
||||
tro.nom_role,
|
||||
'last_name',
|
||||
tro.prenom_role) END,
|
||||
'email', coalesce(borg.email_organisme, tro.email)) AS json_data
|
||||
FROM gn_meta.cor_acquisition_framework_actor cafa
|
||||
LEFT JOIN utilisateurs.bib_organismes borg ON cafa.id_organism = borg.id_organisme
|
||||
LEFT JOIN utilisateurs.t_roles tro ON cafa.id_role = tro.id_role
|
||||
JOIN ref_nomenclatures.t_nomenclatures tn
|
||||
ON cafa.id_nomenclature_actor_role = tn.id_nomenclature
|
||||
), af_territories AS (
|
||||
SELECT
|
||||
caft.id_acquisition_framework,
|
||||
array_agg(DISTINCT t_nomenclatures.cd_nomenclature) AS territories
|
||||
FROM gn_meta.cor_acquisition_framework_territory caft
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures
|
||||
ON caft.id_nomenclature_territory =
|
||||
t_nomenclatures.id_nomenclature
|
||||
GROUP BY caft.id_acquisition_framework
|
||||
), af_objectives AS (
|
||||
SELECT
|
||||
cafo.id_acquisition_framework,
|
||||
array_agg(DISTINCT t_nomenclatures.cd_nomenclature) AS objectives
|
||||
FROM gn_meta.cor_acquisition_framework_objectif cafo
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures
|
||||
ON cafo.id_nomenclature_objectif =
|
||||
t_nomenclatures.id_nomenclature
|
||||
GROUP BY cafo.id_acquisition_framework
|
||||
), af_voletsinp AS (
|
||||
SELECT
|
||||
cafv.id_acquisition_framework,
|
||||
array_agg(DISTINCT t_nomenclatures.cd_nomenclature) AS voletsinp
|
||||
FROM gn_meta.cor_acquisition_framework_voletsinp cafv
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures
|
||||
ON cafv.id_nomenclature_voletsinp =
|
||||
t_nomenclatures.id_nomenclature
|
||||
GROUP BY cafv.id_acquisition_framework
|
||||
), af_publication AS (
|
||||
SELECT
|
||||
cafp.id_acquisition_framework,
|
||||
array_agg(DISTINCT
|
||||
jsonb_build_object('uuid', sinp_datatype_publications.unique_publication_id,
|
||||
'reference',
|
||||
sinp_datatype_publications.publication_reference,
|
||||
'url',
|
||||
sinp_datatype_publications.publication_url)) AS publications
|
||||
FROM gn_meta.cor_acquisition_framework_publication cafp
|
||||
LEFT JOIN gn_meta.sinp_datatype_publications
|
||||
ON cafp.id_publication = sinp_datatype_publications.id_publication
|
||||
GROUP BY cafp.id_acquisition_framework
|
||||
), af AS (
|
||||
SELECT
|
||||
taf.id_acquisition_framework,
|
||||
jsonb_build_object('uuid', taf.unique_acquisition_framework_id, 'name',
|
||||
taf.acquisition_framework_name,
|
||||
'desc', taf.acquisition_framework_desc, 'start_date',
|
||||
taf.acquisition_framework_start_date, 'end_date',
|
||||
taf.acquisition_framework_end_date,
|
||||
'initial_closing_date', taf.initial_closing_date, 'territories',
|
||||
af_territories.territories, 'territorial_level',
|
||||
ntl.cd_nomenclature, 'territory_desc', taf.territory_desc, 'objectives',
|
||||
af_objectives.objectives, 'publications', af_publication.publications,
|
||||
'financing_type', nft.cd_nomenclature,
|
||||
'target_description',
|
||||
taf.target_description, 'ecologic_or_geologic_target',
|
||||
taf.ecologic_or_geologic_target, 'actors',
|
||||
json_agg(af_actors.json_data), 'is_parent', taf.is_parent, 'parent_uuid',
|
||||
tafp.unique_acquisition_framework_id) AS af_data
|
||||
FROM gn_meta.t_acquisition_frameworks taf
|
||||
LEFT JOIN gn_meta.t_acquisition_frameworks tafp
|
||||
ON tafp.id_acquisition_framework = taf.acquisition_framework_parent_id
|
||||
JOIN af_actors ON af_actors.id_acquisition_framework = taf.id_acquisition_framework
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ntl
|
||||
ON taf.id_nomenclature_territorial_level = ntl.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures nft
|
||||
ON taf.id_nomenclature_financing_type = nft.id_nomenclature
|
||||
LEFT JOIN af_territories ON af_territories.id_acquisition_framework = taf.id_acquisition_framework
|
||||
LEFT JOIN af_objectives ON af_objectives.id_acquisition_framework = taf.id_acquisition_framework
|
||||
LEFT JOIN af_voletsinp ON af_voletsinp.id_acquisition_framework = taf.id_acquisition_framework
|
||||
LEFT JOIN af_publication ON af_publication.id_acquisition_framework = taf.id_acquisition_framework
|
||||
GROUP BY taf.id_acquisition_framework, taf.acquisition_framework_name, taf.acquisition_framework_desc,
|
||||
taf.acquisition_framework_start_date, taf.acquisition_framework_end_date, taf.initial_closing_date,
|
||||
ntl.cd_nomenclature, nft.cd_nomenclature, af_territories.territories, af_objectives.objectives,
|
||||
af_voletsinp.voletsinp, af_publication.publications, taf.is_parent,
|
||||
tafp.unique_acquisition_framework_id
|
||||
), ds_actors AS (
|
||||
SELECT
|
||||
cda.id_dataset,
|
||||
json_build_object('type_role',
|
||||
CASE
|
||||
WHEN cda.id_organism IS NOT NULL THEN 'organism'::TEXT
|
||||
WHEN cda.id_role IS NOT NULL THEN 'role'::TEXT
|
||||
ELSE NULL::TEXT
|
||||
END, 'uuid_actor', coalesce(borg.uuid_organisme, tro.uuid_role),
|
||||
'cd_nomenclature_actor_role', tn.cd_nomenclature, 'identity',
|
||||
CASE
|
||||
WHEN cda.id_organism IS NOT NULL
|
||||
THEN json_build_object('organism_name', borg.nom_organisme)
|
||||
WHEN cda.id_role IS NOT NULL THEN json_build_object('first_name',
|
||||
tro.nom_role,
|
||||
'last_name',
|
||||
tro.prenom_role) END,
|
||||
'email', coalesce(borg.email_organisme, tro.email)) AS json_data
|
||||
FROM gn_meta.cor_dataset_actor cda
|
||||
LEFT JOIN utilisateurs.bib_organismes borg ON cda.id_organism = borg.id_organisme
|
||||
LEFT JOIN utilisateurs.t_roles tro ON cda.id_role = tro.id_role
|
||||
JOIN ref_nomenclatures.t_nomenclatures tn
|
||||
ON cda.id_nomenclature_actor_role = tn.id_nomenclature
|
||||
), -- FIN
|
||||
ds_protocols AS (
|
||||
SELECT
|
||||
cdp.id_dataset,
|
||||
jsonb_build_object(
|
||||
'uuid', sdp.unique_protocol_id,
|
||||
'name', sdp.protocol_name,
|
||||
'desc', sdp.protocol_desc,
|
||||
'url', sdp.protocol_url,
|
||||
'type', t_nomenclatures.cd_nomenclature) AS protocols
|
||||
FROM gn_meta.cor_dataset_protocol cdp
|
||||
JOIN gn_meta.sinp_datatype_protocols sdp
|
||||
ON cdp.id_protocol = sdp.id_protocol
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures
|
||||
ON sdp.id_nomenclature_protocol_type = t_nomenclatures.id_nomenclature
|
||||
), ds AS (
|
||||
SELECT
|
||||
tds.id_dataset,
|
||||
tds.id_acquisition_framework,
|
||||
-- tds.additional_data,
|
||||
jsonb_build_object('uuid', tds.unique_dataset_id, 'name', tds.dataset_name, 'desc', tds.dataset_desc,
|
||||
'shortname', tds.dataset_shortname, 'data_type', ndt.cd_nomenclature,
|
||||
'keywords', tds.keywords, 'marine_domain', tds.marine_domain,
|
||||
'terrestrial_domain', tds.terrestrial_domain, 'collecting_method',
|
||||
ncm.cd_nomenclature, 'protocols',
|
||||
ds_protocols.protocols,
|
||||
'data_origin', ndo.cd_nomenclature,
|
||||
'dataset_objectif', ndso.cd_nomenclature, 'resource_type', nrt.cd_nomenclature,
|
||||
'source_status', nss.cd_nomenclature, 'territories', array_agg(DISTINCT
|
||||
ref_nomenclatures.get_cd_nomenclature(cdt.id_nomenclature_territory)),
|
||||
'actors', json_agg(ds_actors.json_data)) AS dataset_data
|
||||
FROM gn_meta.t_datasets tds
|
||||
JOIN ds_actors ON ds_actors.id_dataset = tds.id_dataset
|
||||
LEFT JOIN gn_meta.cor_dataset_territory cdt ON cdt.id_dataset = tds.id_dataset
|
||||
LEFT JOIN ds_protocols ON ds_protocols.id_dataset = tds.id_dataset
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ndt
|
||||
ON tds.id_nomenclature_data_type = ndt.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ncm
|
||||
ON tds.id_nomenclature_collecting_method = ncm.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ndo
|
||||
ON tds.id_nomenclature_data_origin = ndo.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ndso
|
||||
ON tds.id_nomenclature_dataset_objectif = ndso.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures nrt
|
||||
ON tds.id_nomenclature_resource_type = nrt.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures nss
|
||||
ON tds.id_nomenclature_source_status = nss.id_nomenclature
|
||||
GROUP BY tds.id_dataset, tds.id_acquisition_framework, tds.unique_dataset_id, tds.dataset_name,
|
||||
tds.dataset_desc, tds.dataset_shortname, ndt.cd_nomenclature, ncm.cd_nomenclature,
|
||||
ndo.cd_nomenclature, ndso.cd_nomenclature, nrt.cd_nomenclature, nss.cd_nomenclature,
|
||||
ds_protocols.protocols
|
||||
)
|
||||
SELECT row_number() over (ORDER BY id_synthese) as gid,
|
||||
s.id_synthese,
|
||||
s.entity_source_pk_value AS id_source,
|
||||
s.unique_id_sinp AS id_perm_sinp,
|
||||
s.unique_id_sinp_grp AS id_perm_grp_sinp,
|
||||
s.date_min AS date_debut,
|
||||
s.date_max AS date_fin,
|
||||
s.cd_nom,
|
||||
s.meta_v_taxref AS version_taxref,
|
||||
s.nom_cite,
|
||||
s.count_min AS nombre_min,
|
||||
s.count_max AS nombre_max,
|
||||
s.altitude_min,
|
||||
s.altitude_max,
|
||||
s.depth_min AS profondeur_min,
|
||||
s.depth_max AS profondeur_max,
|
||||
s.observers AS observateurs,
|
||||
s.determiner AS determinateur,
|
||||
s.validator AS validateur,
|
||||
s.sample_number_proof AS numero_preuve,
|
||||
s.digital_proof AS preuve_numerique,
|
||||
s.non_digital_proof AS preuve_non_numerique,
|
||||
s.comment_context AS comment_releve,
|
||||
s.comment_description AS comment_occurrence,
|
||||
ds.dataset_data AS jdd_data,
|
||||
af.af_data AS ca_data,
|
||||
s.reference_biblio,
|
||||
s.cd_hab AS code_habitat,
|
||||
h.lb_hab_fr AS habitat,
|
||||
s.place_name AS nom_lieu,
|
||||
s.precision,
|
||||
s.additional_data AS donnees_additionnelles,
|
||||
st_astext(s.the_geom_4326) AS wkt_4326,
|
||||
n1.cd_nomenclature AS nature_objet_geo,
|
||||
n2.cd_nomenclature AS type_regroupement,
|
||||
s.grp_method AS methode_regroupement,
|
||||
n3.cd_nomenclature AS comportement,
|
||||
n4.cd_nomenclature AS technique_obs,
|
||||
n5.cd_nomenclature AS statut_biologique,
|
||||
n6.cd_nomenclature AS etat_biologique,
|
||||
n7.cd_nomenclature AS naturalite,
|
||||
n8.cd_nomenclature AS preuve_existante,
|
||||
n9.cd_nomenclature AS precision_diffusion,
|
||||
n10.cd_nomenclature AS stade_vie,
|
||||
n11.cd_nomenclature AS sexe,
|
||||
n12.cd_nomenclature AS objet_denombrement,
|
||||
n13.cd_nomenclature AS type_denombrement,
|
||||
n14.cd_nomenclature AS niveau_sensibilite,
|
||||
n15.cd_nomenclature AS statut_observation,
|
||||
n16.cd_nomenclature AS floutage_dee,
|
||||
n17.cd_nomenclature AS statut_source,
|
||||
n18.cd_nomenclature AS type_info_geo,
|
||||
n19.cd_nomenclature AS methode_determination,
|
||||
n20.cd_nomenclature AS statut_validation,
|
||||
coalesce(s.meta_update_date, s.meta_create_date) AS derniere_action--, s.the_geom_local -- supprimé car bloque GN2PG
|
||||
FROM gn_synthese.synthese s
|
||||
JOIN taxonomie.taxref USING (cd_nom)
|
||||
JOIN ds ON ds.id_dataset = s.id_dataset
|
||||
JOIN af ON ds.id_acquisition_framework = af.id_acquisition_framework
|
||||
LEFT JOIN ref_habitats.habref h ON h.cd_hab = s.cd_hab
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n1 ON s.id_nomenclature_geo_object_nature = n1.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n2 ON s.id_nomenclature_grp_typ = n2.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n3 ON s.id_nomenclature_behaviour = n3.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n4 ON s.id_nomenclature_obs_technique = n4.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n5 ON s.id_nomenclature_bio_status = n5.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n6 ON s.id_nomenclature_bio_condition = n6.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n7 ON s.id_nomenclature_naturalness = n7.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n8 ON s.id_nomenclature_exist_proof = n8.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n9 ON s.id_nomenclature_diffusion_level = n9.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n10 ON s.id_nomenclature_life_stage = n10.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n11 ON s.id_nomenclature_sex = n11.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n12 ON s.id_nomenclature_obj_count = n12.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n13 ON s.id_nomenclature_type_count = n13.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n14 ON s.id_nomenclature_sensitivity = n14.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n15 ON s.id_nomenclature_observation_status = n15.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n16 ON s.id_nomenclature_blurring = n16.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n17 ON s.id_nomenclature_source_status = n17.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n18 ON s.id_nomenclature_info_geo_type = n18.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n19 ON s.id_nomenclature_determination_method = n19.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n20 ON s.id_nomenclature_valid_status = n20.id_nomenclature
|
||||
|
||||
WHERE (af.af_data#>>'{uuid}') = '97c05888-08f5-4a4b-a224-20c3263594ad'
|
||||
AND (ds.dataset_data#>>'{uuid}') not in (
|
||||
'7a29acae-757e-41d7-a0cf-a35c568a5db9',
|
||||
'089a8dc1-3d50-4b2c-865c-d535a982446f' -- JDD : Expertise écologique - Label Golf pour la biodiversité
|
||||
)
|
||||
-- FIN
|
||||
ORDER BY s.id_synthese;
|
||||
'''
|
||||
|
||||
with con_gn.begin() as cnx:
|
||||
cnx.execute(sql)
|
||||
@@ -0,0 +1,45 @@
|
||||
from pycen import con_gn
|
||||
|
||||
sql = '''
|
||||
DROP VIEW IF EXISTS gn_exports.v_synthese_zh_rhomeosite;
|
||||
|
||||
CREATE OR REPLACE VIEW gn_exports.v_synthese_zh_rhomeosite
|
||||
AS WITH t1 AS (
|
||||
SELECT t_zh.code,
|
||||
l_areas.area_name,
|
||||
st_area(st_intersection(t_zh.geom, l_areas.geom_4326)) AS area_intersect
|
||||
FROM pr_zh.t_zh
|
||||
JOIN ref_geo.l_areas ON st_intersects(t_zh.geom, l_areas.geom_4326)
|
||||
JOIN ref_geo.bib_areas_types bib ON l_areas.id_type = bib.id_type and bib.type_name = 'Zones biogéographiques'
|
||||
), t2 AS (
|
||||
SELECT t1.code,
|
||||
t1.area_name,
|
||||
row_number() OVER (PARTITION BY t1.code ORDER BY t1.area_intersect DESC) AS ismax_ter
|
||||
FROM t1
|
||||
)
|
||||
SELECT
|
||||
t_zh.id_zh,
|
||||
t_zh.main_name AS "NAME",
|
||||
concat(t_roles.nom_role, ' ', t_roles.prenom_role) AS "REFERENT",
|
||||
bib.nom_organisme AS "ORG",
|
||||
CASE
|
||||
WHEN t_zh.id_sage IS NULL
|
||||
THEN ref_nomenclatures.get_cd_nomenclature(t_zh.id_sdage)::text
|
||||
ELSE ref_nomenclatures.get_cd_nomenclature(t_zh.id_sage)::text
|
||||
END AS "TYPE",
|
||||
CASE
|
||||
WHEN t2.area_name = 'alpin'::text THEN '1'::text
|
||||
WHEN t2.area_name = 'continental'::text THEN '2'::text
|
||||
WHEN t2.area_name = 'mediterraneen'::text THEN '4'::text
|
||||
ELSE NULL::text
|
||||
END AS "ODONATE",
|
||||
ST_TRANSFORM(t_zh.geom,2154)::geometry(geometry,2154) geom
|
||||
FROM pr_zh.t_zh
|
||||
LEFT JOIN utilisateurs.t_roles ON t_zh.update_author = t_roles.id_role
|
||||
LEFT JOIN utilisateurs.bib_organismes bib USING (id_organisme)
|
||||
JOIN t2 ON t_zh.code::text = t2.code::text AND t2.ismax_ter = 1;
|
||||
'''
|
||||
|
||||
with con_gn.begin() as cnx:
|
||||
cnx.execute(sql)
|
||||
|
||||
@@ -0,0 +1,391 @@
|
||||
from pycen import con_gn
|
||||
|
||||
sql0 = '''
|
||||
with habitats as (
|
||||
SELECT
|
||||
zh_uuid,
|
||||
json_agg(lb_code) AS habitats
|
||||
FROM pr_zh.t_zh
|
||||
LEFT JOIN pr_zh.cor_zh_cb USING (id_zh)
|
||||
group by 1 order by 1
|
||||
), crit_delim as (
|
||||
SELECT
|
||||
zh_uuid,
|
||||
json_agg(ref_nomenclatures.get_nomenclature_label(cor_lim_list.id_lim)) AS crit_delim
|
||||
FROM pr_zh.t_zh
|
||||
LEFT JOIN pr_zh.cor_lim_list USING (id_lim_list)
|
||||
group by 1 order by 1
|
||||
), esp_fs as (
|
||||
SELECT
|
||||
zh_uuid,
|
||||
json_agg(ref_nomenclatures.get_nomenclature_label(cor_zh_lim_fs.id_lim_fs)) AS crit_esp_fs
|
||||
FROM pr_zh.t_zh
|
||||
LEFT JOIN pr_zh.cor_zh_lim_fs USING (id_zh)
|
||||
group by 1 order by 1
|
||||
), v_activity AS (
|
||||
SELECT
|
||||
id_zh,
|
||||
json_agg(ref_nomenclatures.get_nomenclature_label(cor_impact_types.id_impact)) impacts
|
||||
FROM pr_zh.t_activity
|
||||
LEFT JOIN pr_zh.cor_impact_list USING (id_impact_list)
|
||||
LEFT JOIN pr_zh.cor_impact_types USING (id_cor_impact_types)
|
||||
group by 1 order by 1
|
||||
), activ_hum AS (
|
||||
SELECT
|
||||
zh_uuid,
|
||||
json_agg(CASE WHEN t_activity.id_activity IS NULL
|
||||
THEN NULL
|
||||
ELSE json_build_object(
|
||||
'activite', ref_nomenclatures.get_nomenclature_label(t_activity.id_activity),
|
||||
'position', ref_nomenclatures.get_nomenclature_label(t_activity.id_position),
|
||||
'impact', v_activity.impacts,
|
||||
'rmq_activ', t_activity.remark_activity
|
||||
)
|
||||
END) AS activ_hum
|
||||
FROM pr_zh.t_zh
|
||||
LEFT JOIN pr_zh.t_activity USING (id_zh)
|
||||
LEFT JOIN v_activity USING (id_zh)
|
||||
group by zh_uuid order by 1
|
||||
--), activ_hum_test AS (
|
||||
-- SELECT
|
||||
-- zh_uuid,
|
||||
-- CASE WHEN t_activity.id_activity IS NOT NULL
|
||||
-- THEN jsonb_build_object(
|
||||
-- ref_nomenclatures.get_nomenclature_label(t_activity.id_activity), v_activity.impacts
|
||||
-- )
|
||||
-- ELSE NULL
|
||||
-- END AS activ_hum_test
|
||||
-- FROM pr_zh.t_zh
|
||||
-- LEFT JOIN pr_zh.t_activity USING (id_zh)
|
||||
-- LEFT JOIN v_activity USING (id_zh)
|
||||
-- --group by zh_uuid order by 1
|
||||
), inflow as (
|
||||
SELECT
|
||||
zh_uuid,
|
||||
json_agg(CASE WHEN t_inflow.id_inflow IS NULL
|
||||
THEN NULL
|
||||
ELSE json_build_object(
|
||||
'entree_eau', ref_nomenclatures.get_nomenclature_label(t_inflow.id_inflow),
|
||||
'perm', ref_nomenclatures.get_nomenclature_label(t_inflow.id_permanance),
|
||||
'topo', t_inflow.topo
|
||||
)
|
||||
END) AS entree_eau
|
||||
FROM pr_zh.t_zh
|
||||
LEFT JOIN pr_zh.t_inflow USING (id_zh)
|
||||
group by zh_uuid
|
||||
), outflow as (
|
||||
SELECT
|
||||
zh_uuid,
|
||||
json_agg(CASE WHEN t_outflow.id_outflow IS NULL
|
||||
THEN NULL
|
||||
ELSE json_build_object(
|
||||
'sortie_eau', ref_nomenclatures.get_nomenclature_label(t_outflow.id_outflow),
|
||||
'perm', ref_nomenclatures.get_nomenclature_label(t_outflow.id_permanance),
|
||||
'topo', t_outflow.topo
|
||||
)
|
||||
END) AS sortie_eau
|
||||
FROM pr_zh.t_zh
|
||||
LEFT JOIN pr_zh.t_outflow USING (id_zh)
|
||||
group by zh_uuid
|
||||
), functions as (
|
||||
WITH hydro as (
|
||||
SELECT
|
||||
zh_uuid,
|
||||
json_agg(json_build_object(
|
||||
'fonctions', ref_nomenclatures.get_nomenclature_label(t_functions.id_function),
|
||||
'qualification', ref_nomenclatures.get_nomenclature_label(t_functions.id_qualification),
|
||||
'connaissance', ref_nomenclatures.get_nomenclature_label(t_functions.id_knowledge),
|
||||
'justification', t_functions.justification
|
||||
)) AS hydro
|
||||
FROM pr_zh.t_zh
|
||||
LEFT JOIN pr_zh.t_functions USING (id_zh)
|
||||
WHERE id_function IN (
|
||||
SELECT
|
||||
id_nomenclature
|
||||
FROM ref_nomenclatures.t_nomenclatures
|
||||
JOIN ref_nomenclatures.bib_nomenclatures_types bib USING (id_type)
|
||||
WHERE bib.mnemonique = 'FONCTIONS_HYDRO')
|
||||
group by zh_uuid
|
||||
), bio as (
|
||||
SELECT
|
||||
zh_uuid,
|
||||
json_agg(json_build_object(
|
||||
'fonctions', ref_nomenclatures.get_nomenclature_label(t_functions.id_function),
|
||||
'qualification', ref_nomenclatures.get_nomenclature_label(t_functions.id_qualification),
|
||||
'connaissance', ref_nomenclatures.get_nomenclature_label(t_functions.id_knowledge),
|
||||
'justification', t_functions.justification
|
||||
)) AS bio
|
||||
FROM pr_zh.t_zh
|
||||
LEFT JOIN pr_zh.t_functions USING (id_zh)
|
||||
WHERE id_function IN (
|
||||
SELECT
|
||||
id_nomenclature
|
||||
FROM ref_nomenclatures.t_nomenclatures
|
||||
JOIN ref_nomenclatures.bib_nomenclatures_types bib USING (id_type)
|
||||
WHERE bib.mnemonique = 'FONCTIONS_BIO')
|
||||
group by zh_uuid
|
||||
), int_patrim as (
|
||||
SELECT
|
||||
zh_uuid,
|
||||
json_agg(json_build_object(
|
||||
'fonctions', ref_nomenclatures.get_nomenclature_label(t_functions.id_function),
|
||||
'qualification', ref_nomenclatures.get_nomenclature_label(t_functions.id_qualification),
|
||||
'connaissance', ref_nomenclatures.get_nomenclature_label(t_functions.id_knowledge),
|
||||
'justification', t_functions.justification
|
||||
)) AS int_patrim
|
||||
FROM pr_zh.t_zh
|
||||
LEFT JOIN pr_zh.t_functions USING (id_zh)
|
||||
WHERE id_function IN (
|
||||
SELECT
|
||||
id_nomenclature
|
||||
FROM ref_nomenclatures.t_nomenclatures
|
||||
JOIN ref_nomenclatures.bib_nomenclatures_types bib USING (id_type)
|
||||
WHERE bib.mnemonique = 'INTERET_PATRIM')
|
||||
group by zh_uuid
|
||||
), val_socio as (
|
||||
SELECT
|
||||
zh_uuid,
|
||||
json_agg(json_build_object(
|
||||
'fonctions', ref_nomenclatures.get_nomenclature_label(t_functions.id_function),
|
||||
'qualification', ref_nomenclatures.get_nomenclature_label(t_functions.id_qualification),
|
||||
'connaissance', ref_nomenclatures.get_nomenclature_label(t_functions.id_knowledge),
|
||||
'justification', t_functions.justification
|
||||
)) AS val_socio
|
||||
FROM pr_zh.t_zh
|
||||
LEFT JOIN pr_zh.t_functions USING (id_zh)
|
||||
WHERE id_function IN (
|
||||
SELECT
|
||||
id_nomenclature
|
||||
FROM ref_nomenclatures.t_nomenclatures
|
||||
JOIN ref_nomenclatures.bib_nomenclatures_types bib USING (id_type)
|
||||
WHERE bib.mnemonique = 'VAL_SOC_ECO')
|
||||
group by zh_uuid
|
||||
)
|
||||
SELECT
|
||||
zh_uuid,
|
||||
hydro.hydro,
|
||||
bio.bio,
|
||||
int_patrim.int_patrim,
|
||||
val_socio.val_socio
|
||||
FROM pr_zh.t_zh
|
||||
LEFT JOIN hydro USING (zh_uuid)
|
||||
LEFT JOIN bio USING (zh_uuid)
|
||||
LEFT JOIN int_patrim USING (zh_uuid)
|
||||
LEFT JOIN val_socio USING (zh_uuid)
|
||||
--group by zh_uuid
|
||||
), especes as (
|
||||
SELECT
|
||||
zh_uuid,
|
||||
(select distinct count(vm_flora.cd_nom) from pr_zh.vm_flora where id_zh = t.id_zh) nb_flora,
|
||||
(select distinct count(vm_vertebrates.cd_nom) from pr_zh.vm_vertebrates where id_zh = t.id_zh) nb_vertebrates,
|
||||
(select distinct count(vm_invertebrates.cd_nom) from pr_zh.vm_invertebrates where id_zh = t.id_zh) nb_invertebrates
|
||||
--count(vm_vertebrates.cd_nom) nb_vertebrates,
|
||||
--count(vm_invertebrates.cd_nom) nb_invertebrates
|
||||
FROM pr_zh.t_zh t
|
||||
--GROUP BY zh_uuid
|
||||
), rb as (
|
||||
SELECT
|
||||
id_zh,
|
||||
json_agg(name) river_bassin
|
||||
FROM pr_zh.cor_zh_rb
|
||||
JOIN pr_zh.t_river_basin USING (id_rb)
|
||||
GROUP BY id_zh
|
||||
), instrument as (
|
||||
SELECT
|
||||
id_zh,
|
||||
json_agg(ref_nomenclatures.get_nomenclature_label(t_instruments.id_instrument)) instrument
|
||||
FROM pr_zh.t_instruments
|
||||
GROUP BY id_zh
|
||||
), urba as (
|
||||
WITH urban_type as (
|
||||
SELECT
|
||||
id_doc,
|
||||
json_agg(json_build_object(
|
||||
'typ_doc', ref_nomenclatures.get_nomenclature_label(cor_urban_type_range.id_doc_type),
|
||||
'zone', ref_nomenclatures.get_nomenclature_label(cor_urban_type_range.id_range_type)
|
||||
)) doc_range
|
||||
FROM pr_zh.cor_zh_doc_range
|
||||
JOIN pr_zh.cor_urban_type_range USING (id_cor)
|
||||
GROUP BY id_doc
|
||||
)
|
||||
SELECT
|
||||
id_zh,
|
||||
json_agg(json_build_object(
|
||||
'typ_doc', ref_nomenclatures.get_nomenclature_label(t_urban_planning_docs.id_doc_type),
|
||||
'doc_range', doc_range,
|
||||
'zone', l_areas.area_name,
|
||||
'remarque', t_urban_planning_docs.remark
|
||||
)) docs_urba
|
||||
FROM pr_zh.t_urban_planning_docs
|
||||
JOIN ref_geo.l_areas USING (id_area)
|
||||
JOIN urban_type USING (id_doc)
|
||||
GROUP BY id_zh
|
||||
), actions AS (
|
||||
SELECT
|
||||
id_zh,
|
||||
json_agg(json_build_object(
|
||||
'priorite', ref_nomenclatures.get_nomenclature_label(t_actions.id_priority_level),
|
||||
'action', bib_actions.name,
|
||||
'remarque', t_actions.remark
|
||||
) ORDER BY id_priority_level,bib_actions.name) prio_actions
|
||||
FROM pr_zh.t_actions
|
||||
JOIN pr_zh.bib_actions USING (id_action)
|
||||
GROUp BY id_zh
|
||||
), zh_area_com AS (
|
||||
SELECT
|
||||
id_zh,
|
||||
json_agg(l_areas.area_name||' ('||cover||'%%)' ORDER BY l_areas.area_name) commune
|
||||
FROM pr_zh.cor_zh_area
|
||||
JOIN ref_geo.l_areas USING (id_area)
|
||||
WHERE cover IS NOT NULL
|
||||
GROUP BY id_zh
|
||||
), dep AS (
|
||||
SELECT
|
||||
id_zh,
|
||||
json_agg(l_areas.area_name ORDER BY l_areas.area_name) departement
|
||||
FROM pr_zh.cor_zh_area
|
||||
JOIN ref_geo.l_areas USING (id_area)
|
||||
join ref_geo.bib_areas_types bat using (id_type)
|
||||
WHERE cover IS null and bat.type_code = 'DEP'
|
||||
GROUP BY id_zh
|
||||
), inventaire AS (
|
||||
SELECT
|
||||
id_zh,
|
||||
json_agg(json_build_object(
|
||||
'inventaire',l_areas.area_code ||' - '||l_areas.area_name,
|
||||
'url', l_areas."source"
|
||||
) ORDER BY l_areas.area_name) inventaire
|
||||
FROM pr_zh.cor_zh_area
|
||||
JOIN ref_geo.l_areas USING (id_area)
|
||||
join ref_geo.bib_areas_types bat using (id_type)
|
||||
WHERE cover IS null and bat.type_code not in ('DEP')
|
||||
GROUP BY id_zh
|
||||
)
|
||||
SELECT
|
||||
zh_uuid,
|
||||
code,
|
||||
main_name,
|
||||
--concat(crea_role.nom_role, ' ', crea_role.prenom_role) AS crea_author,
|
||||
--bib_org.name AS organisme,
|
||||
--concat(updt_role.nom_role, ' ', updt_role.prenom_role) AS updt_author,
|
||||
bib_org_user_updt.nom_organisme as org_updt,
|
||||
create_date crea_date,
|
||||
update_date updt_date,
|
||||
--ST_TRANSFORM(geom,2154)::geometry(geometry,2154) geom,
|
||||
geom,
|
||||
area surface,
|
||||
rb.river_bassin,
|
||||
dep.departement,
|
||||
zh_area_com.commune,
|
||||
ref_nomenclatures.get_nomenclature_label(id_sdage) sdage,
|
||||
ref_nomenclatures.get_nomenclature_label(id_sage) sage,
|
||||
CASE WHEN habitats.habitats::text = '[null]'
|
||||
THEN NULL
|
||||
ELSE habitats.habitats
|
||||
END AS habitats,
|
||||
remark_pres descript,
|
||||
crit_delim.crit_delim,
|
||||
remark_lim rmq_delilm,
|
||||
CASE WHEN esp_fs.crit_esp_fs::text = '[null]'
|
||||
THEN NULL
|
||||
ELSE esp_fs.crit_esp_fs
|
||||
END AS crit_esp_fs,
|
||||
remark_lim_fs rmq_esp_fs,
|
||||
--ef_area,
|
||||
--v_habref v_hab,
|
||||
CASE WHEN activ_hum.activ_hum::text = '[null]'
|
||||
THEN NULL
|
||||
ELSE activ_hum.activ_hum::text
|
||||
END AS activ_hum,
|
||||
-- activ_hum_test.activ_hum_test,
|
||||
ref_nomenclatures.get_nomenclature_label(id_thread) bilan_menace,
|
||||
CASE WHEN inflow.entree_eau::text = '[null]'
|
||||
THEN NULL
|
||||
ELSE inflow.entree_eau::text
|
||||
END AS entree_eau,
|
||||
CASE WHEN outflow.sortie_eau::text = '[null]'
|
||||
THEN NULL
|
||||
ELSE outflow.sortie_eau::text
|
||||
END AS sortie_eau,
|
||||
ref_nomenclatures.get_nomenclature_label(id_frequency) freq_sub,
|
||||
ref_nomenclatures.get_nomenclature_label(id_spread) extend_sub,
|
||||
ref_nomenclatures.get_nomenclature_label(id_connexion) connex,
|
||||
ref_nomenclatures.get_nomenclature_label(id_diag_hydro) diag_hydro,
|
||||
ref_nomenclatures.get_nomenclature_label(id_diag_bio) diag_bio,
|
||||
functions.hydro::text,
|
||||
functions.bio::text,
|
||||
functions.int_patrim::text,
|
||||
functions.val_socio::text,
|
||||
remark_eval_functions eval_fcts,
|
||||
remark_eval_heritage eval_heritage,
|
||||
remark_eval_thread eval_menace,
|
||||
remark_eval_actions eval_actions,
|
||||
ref_nomenclatures.get_nomenclature_label(id_strat_gestion) strat_gestion,
|
||||
remark_diag rmq_gestion,
|
||||
--is_other_inventory,
|
||||
--is_carto_hab,
|
||||
--nb_hab,
|
||||
--total_hab_cover,
|
||||
especes.nb_flora nb_flora_sp,
|
||||
especes.nb_vertebrates nb_vertebrate_sp,
|
||||
especes.nb_invertebrates nb_invertebrate_sp,
|
||||
inventaire.inventaire::text,
|
||||
actions.prio_actions::text,
|
||||
instrument.instrument::text,
|
||||
urba.docs_urba::text
|
||||
--remark_is_other_inventory
|
||||
FROM pr_zh.t_zh
|
||||
JOIN utilisateurs.t_roles crea_role ON t_zh.create_author = crea_role.id_role
|
||||
LEFT JOIN utilisateurs.t_roles updt_role ON t_zh.update_author = updt_role.id_role
|
||||
LEFT JOIN utilisateurs.bib_organismes bib_org_user_updt on updt_role.id_organisme = bib_org_user_updt.id_organisme
|
||||
LEFT JOIN pr_zh.bib_organismes bib_org USING (id_org)
|
||||
LEFT JOIN habitats USING (zh_uuid)
|
||||
LEFT JOIN crit_delim USING (zh_uuid)
|
||||
LEFT JOIN esp_fs USING (zh_uuid)
|
||||
LEFT JOIN activ_hum USING (zh_uuid)
|
||||
--LEFT JOIN activ_hum_test USING (zh_uuid)
|
||||
LEFT JOIN inflow USING (zh_uuid)
|
||||
LEFT JOIN outflow USING (zh_uuid)
|
||||
LEFT JOIN functions USING (zh_uuid)
|
||||
LEFT JOIN especes USING (zh_uuid)
|
||||
LEFT JOIN rb USING (id_zh)
|
||||
LEFT JOIN instrument USING (id_zh)
|
||||
LEFT JOIN urba USING (id_zh)
|
||||
LEFT JOIN actions USING (id_zh)
|
||||
LEFT JOIN zh_area_com USING (id_zh)
|
||||
LEFT JOIN dep USING (id_zh)
|
||||
LEFT JOIN inventaire USING (id_zh)
|
||||
'''
|
||||
|
||||
sql_zh_all = '''
|
||||
DROP VIEW IF EXISTS gn_exports.v_synthese_zones_humides;
|
||||
|
||||
CREATE VIEW gn_exports.v_synthese_zones_humides AS
|
||||
''' + sql0 + '''
|
||||
--group BY zh_uuid
|
||||
WHERE code not like '38CENIS%%'
|
||||
AND ref_nomenclatures.get_cd_nomenclature(id_sdage)::int > 0
|
||||
--id_sdage > 0
|
||||
order by code
|
||||
;
|
||||
'''
|
||||
|
||||
with con_gn.begin() as cnx:
|
||||
cnx.execute(sql_zh_all)
|
||||
|
||||
|
||||
sql_zh_38 = '''
|
||||
DROP VIEW IF EXISTS gn_exports.v_synthese_zones_humides_isere;
|
||||
|
||||
CREATE VIEW gn_exports.v_synthese_zones_humides_isere AS
|
||||
''' + sql0 + '''
|
||||
--group BY zh_uuid
|
||||
WHERE code not like '38CENIS%%'
|
||||
AND dep.departement::jsonb @> '"Isère"'
|
||||
AND ref_nomenclatures.get_cd_nomenclature(id_sdage)::int > 0
|
||||
order by code
|
||||
;
|
||||
'''
|
||||
|
||||
with con_gn.begin() as cnx:
|
||||
cnx.execute(sql_zh_38)
|
||||
@@ -0,0 +1,276 @@
|
||||
from pycen import con_gn
|
||||
|
||||
sql = '''
|
||||
DROP VIEW IF EXISTS gn_exports.v_synthese_sinp_with_metadata_vertebre_for_gn2pg;
|
||||
|
||||
CREATE VIEW gn_exports.v_synthese_sinp_with_metadata_vertebre_for_gn2pg AS
|
||||
WITH af_actors AS (
|
||||
SELECT cafa.id_acquisition_framework,
|
||||
json_build_object('type_role',
|
||||
CASE
|
||||
WHEN cafa.id_organism IS NOT NULL THEN 'organism'::TEXT
|
||||
WHEN cafa.id_role IS NOT NULL THEN 'role'::TEXT
|
||||
ELSE NULL::TEXT
|
||||
END, 'uuid_actor', coalesce(borg.uuid_organisme, tro.uuid_role),
|
||||
'cd_nomenclature_actor_role', tn.cd_nomenclature, 'identity',
|
||||
CASE
|
||||
WHEN cafa.id_organism IS NOT NULL
|
||||
THEN json_build_object('organism_name', borg.nom_organisme)
|
||||
WHEN cafa.id_role IS NOT NULL
|
||||
THEN json_build_object('first_name',
|
||||
tro.nom_role,
|
||||
'last_name',
|
||||
tro.prenom_role) END,
|
||||
'email', coalesce(borg.email_organisme, tro.email)) AS json_data
|
||||
FROM gn_meta.cor_acquisition_framework_actor cafa
|
||||
LEFT JOIN utilisateurs.bib_organismes borg ON cafa.id_organism = borg.id_organisme
|
||||
LEFT JOIN utilisateurs.t_roles tro ON cafa.id_role = tro.id_role
|
||||
JOIN ref_nomenclatures.t_nomenclatures tn
|
||||
ON cafa.id_nomenclature_actor_role = tn.id_nomenclature),
|
||||
af_territories AS (SELECT caft.id_acquisition_framework,
|
||||
array_agg(DISTINCT t_nomenclatures.cd_nomenclature) AS territories
|
||||
FROM gn_meta.cor_acquisition_framework_territory caft
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures
|
||||
ON caft.id_nomenclature_territory =
|
||||
t_nomenclatures.id_nomenclature
|
||||
GROUP BY caft.id_acquisition_framework),
|
||||
af_objectives AS (SELECT cafo.id_acquisition_framework,
|
||||
array_agg(DISTINCT t_nomenclatures.cd_nomenclature) AS objectives
|
||||
FROM gn_meta.cor_acquisition_framework_objectif cafo
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures
|
||||
ON cafo.id_nomenclature_objectif =
|
||||
t_nomenclatures.id_nomenclature
|
||||
GROUP BY cafo.id_acquisition_framework),
|
||||
af_voletsinp AS (SELECT cafv.id_acquisition_framework,
|
||||
array_agg(DISTINCT t_nomenclatures.cd_nomenclature) AS voletsinp
|
||||
FROM gn_meta.cor_acquisition_framework_voletsinp cafv
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures
|
||||
ON cafv.id_nomenclature_voletsinp =
|
||||
t_nomenclatures.id_nomenclature
|
||||
GROUP BY cafv.id_acquisition_framework),
|
||||
af_publication AS (SELECT cafp.id_acquisition_framework,
|
||||
array_agg(DISTINCT
|
||||
jsonb_build_object('uuid', sinp_datatype_publications.unique_publication_id,
|
||||
'reference',
|
||||
sinp_datatype_publications.publication_reference,
|
||||
'url',
|
||||
sinp_datatype_publications.publication_url)) AS publications
|
||||
FROM gn_meta.cor_acquisition_framework_publication cafp
|
||||
LEFT JOIN gn_meta.sinp_datatype_publications
|
||||
ON cafp.id_publication = sinp_datatype_publications.id_publication
|
||||
GROUP BY cafp.id_acquisition_framework)
|
||||
,
|
||||
af AS (SELECT taf.id_acquisition_framework,
|
||||
jsonb_build_object('uuid', taf.unique_acquisition_framework_id, 'name',
|
||||
taf.acquisition_framework_name,
|
||||
'desc', taf.acquisition_framework_desc, 'start_date',
|
||||
taf.acquisition_framework_start_date, 'end_date',
|
||||
taf.acquisition_framework_end_date,
|
||||
'initial_closing_date', taf.initial_closing_date, 'territories',
|
||||
af_territories.territories, 'territorial_level',
|
||||
ntl.cd_nomenclature, 'territory_desc', taf.territory_desc, 'objectives',
|
||||
af_objectives.objectives, 'publications', af_publication.publications,
|
||||
'financing_type', nft.cd_nomenclature,
|
||||
'target_description',
|
||||
taf.target_description, 'ecologic_or_geologic_target',
|
||||
taf.ecologic_or_geologic_target, 'actors',
|
||||
json_agg(af_actors.json_data), 'is_parent', taf.is_parent, 'parent_uuid',
|
||||
tafp.unique_acquisition_framework_id) AS af_data
|
||||
FROM gn_meta.t_acquisition_frameworks taf
|
||||
LEFT JOIN gn_meta.t_acquisition_frameworks tafp
|
||||
ON tafp.id_acquisition_framework = taf.acquisition_framework_parent_id
|
||||
JOIN af_actors ON af_actors.id_acquisition_framework = taf.id_acquisition_framework
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ntl
|
||||
ON taf.id_nomenclature_territorial_level = ntl.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures nft
|
||||
ON taf.id_nomenclature_financing_type = nft.id_nomenclature
|
||||
LEFT JOIN af_territories ON af_territories.id_acquisition_framework = taf.id_acquisition_framework
|
||||
LEFT JOIN af_objectives ON af_objectives.id_acquisition_framework = taf.id_acquisition_framework
|
||||
LEFT JOIN af_voletsinp ON af_voletsinp.id_acquisition_framework = taf.id_acquisition_framework
|
||||
LEFT JOIN af_publication ON af_publication.id_acquisition_framework = taf.id_acquisition_framework
|
||||
GROUP BY taf.id_acquisition_framework, taf.acquisition_framework_name, taf.acquisition_framework_desc,
|
||||
taf.acquisition_framework_start_date, taf.acquisition_framework_end_date, taf.initial_closing_date,
|
||||
ntl.cd_nomenclature, nft.cd_nomenclature, af_territories.territories, af_objectives.objectives,
|
||||
af_voletsinp.voletsinp, af_publication.publications, taf.is_parent,
|
||||
tafp.unique_acquisition_framework_id),
|
||||
ds_actors AS (SELECT cda.id_dataset,
|
||||
json_build_object('type_role',
|
||||
CASE
|
||||
WHEN cda.id_organism IS NOT NULL THEN 'organism'::TEXT
|
||||
WHEN cda.id_role IS NOT NULL THEN 'role'::TEXT
|
||||
ELSE NULL::TEXT
|
||||
END, 'uuid_actor', coalesce(borg.uuid_organisme, tro.uuid_role),
|
||||
'cd_nomenclature_actor_role', tn.cd_nomenclature, 'identity',
|
||||
CASE
|
||||
WHEN cda.id_organism IS NOT NULL
|
||||
THEN json_build_object('organism_name', borg.nom_organisme)
|
||||
WHEN cda.id_role IS NOT NULL THEN json_build_object('first_name',
|
||||
tro.nom_role,
|
||||
'last_name',
|
||||
tro.prenom_role) END,
|
||||
'email', coalesce(borg.email_organisme, tro.email)) AS json_data
|
||||
FROM gn_meta.cor_dataset_actor cda
|
||||
LEFT JOIN utilisateurs.bib_organismes borg ON cda.id_organism = borg.id_organisme
|
||||
LEFT JOIN utilisateurs.t_roles tro ON cda.id_role = tro.id_role
|
||||
JOIN ref_nomenclatures.t_nomenclatures tn
|
||||
ON cda.id_nomenclature_actor_role = tn.id_nomenclature),
|
||||
ds_protocols AS (SELECT cdp.id_dataset,
|
||||
jsonb_build_object(
|
||||
'uuid', sdp.unique_protocol_id,
|
||||
'name', sdp.protocol_name,
|
||||
'desc', sdp.protocol_desc,
|
||||
'url', sdp.protocol_url,
|
||||
'type', t_nomenclatures.cd_nomenclature) AS protocols
|
||||
FROM gn_meta.cor_dataset_protocol cdp
|
||||
JOIN gn_meta.sinp_datatype_protocols sdp
|
||||
ON cdp.id_protocol = sdp.id_protocol
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures
|
||||
ON sdp.id_nomenclature_protocol_type = t_nomenclatures.id_nomenclature)
|
||||
,
|
||||
ds AS (SELECT tds.id_dataset,
|
||||
tds.id_acquisition_framework,
|
||||
-- tds.additional_data,
|
||||
jsonb_build_object('uuid', tds.unique_dataset_id, 'name', tds.dataset_name, 'desc', tds.dataset_desc,
|
||||
'shortname', tds.dataset_shortname, 'data_type', ndt.cd_nomenclature,
|
||||
'keywords', tds.keywords, 'marine_domain', tds.marine_domain,
|
||||
'terrestrial_domain', tds.terrestrial_domain, 'collecting_method',
|
||||
ncm.cd_nomenclature, 'protocols',
|
||||
ds_protocols.protocols,
|
||||
'data_origin', ndo.cd_nomenclature,
|
||||
'dataset_objectif', ndso.cd_nomenclature, 'resource_type', nrt.cd_nomenclature,
|
||||
'source_status', nss.cd_nomenclature, 'territories', array_agg(DISTINCT
|
||||
ref_nomenclatures.get_cd_nomenclature(cdt.id_nomenclature_territory)),
|
||||
'actors', json_agg(ds_actors.json_data)) AS dataset_data
|
||||
FROM gn_meta.t_datasets tds
|
||||
JOIN ds_actors ON ds_actors.id_dataset = tds.id_dataset
|
||||
LEFT JOIN gn_meta.cor_dataset_territory cdt ON cdt.id_dataset = tds.id_dataset
|
||||
LEFT JOIN ds_protocols ON ds_protocols.id_dataset = tds.id_dataset
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ndt
|
||||
ON tds.id_nomenclature_data_type = ndt.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ncm
|
||||
ON tds.id_nomenclature_collecting_method = ncm.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ndo
|
||||
ON tds.id_nomenclature_data_origin = ndo.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures ndso
|
||||
ON tds.id_nomenclature_dataset_objectif = ndso.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures nrt
|
||||
ON tds.id_nomenclature_resource_type = nrt.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures nss
|
||||
ON tds.id_nomenclature_source_status = nss.id_nomenclature
|
||||
GROUP BY tds.id_dataset, tds.id_acquisition_framework, tds.unique_dataset_id, tds.dataset_name,
|
||||
tds.dataset_desc, tds.dataset_shortname, ndt.cd_nomenclature, ncm.cd_nomenclature,
|
||||
ndo.cd_nomenclature, ndso.cd_nomenclature, nrt.cd_nomenclature, nss.cd_nomenclature,
|
||||
ds_protocols.protocols)
|
||||
SELECT s.id_synthese,
|
||||
s.entity_source_pk_value AS id_source,
|
||||
s.unique_id_sinp AS id_perm_sinp,
|
||||
s.unique_id_sinp_grp AS id_perm_grp_sinp,
|
||||
s.date_min AS date_debut,
|
||||
s.date_max AS date_fin,
|
||||
s.cd_nom,
|
||||
s.meta_v_taxref AS version_taxref,
|
||||
s.nom_cite,
|
||||
s.count_min AS nombre_min,
|
||||
s.count_max AS nombre_max,
|
||||
s.altitude_min,
|
||||
s.altitude_max,
|
||||
s.depth_min AS profondeur_min,
|
||||
s.depth_max AS profondeur_max,
|
||||
s.observers AS observateurs,
|
||||
s.determiner AS determinateur,
|
||||
s.validator AS validateur,
|
||||
s.sample_number_proof AS numero_preuve,
|
||||
s.digital_proof AS preuve_numerique,
|
||||
s.non_digital_proof AS preuve_non_numerique,
|
||||
s.comment_context AS comment_releve,
|
||||
s.comment_description AS comment_occurrence,
|
||||
ds.dataset_data AS jdd_data,
|
||||
af.af_data AS ca_data,
|
||||
s.reference_biblio,
|
||||
s.cd_hab AS code_habitat,
|
||||
h.lb_hab_fr AS habitat,
|
||||
s.place_name AS nom_lieu,
|
||||
s.precision,
|
||||
s.additional_data AS donnees_additionnelles,
|
||||
st_astext(s.the_geom_4326) AS wkt_4326,
|
||||
n1.cd_nomenclature AS nature_objet_geo,
|
||||
n2.cd_nomenclature AS type_regroupement,
|
||||
s.grp_method AS methode_regroupement,
|
||||
n3.cd_nomenclature AS comportement,
|
||||
n4.cd_nomenclature AS technique_obs,
|
||||
n5.cd_nomenclature AS statut_biologique,
|
||||
n6.cd_nomenclature AS etat_biologique,
|
||||
n7.cd_nomenclature AS naturalite,
|
||||
n8.cd_nomenclature AS preuve_existante,
|
||||
n9.cd_nomenclature AS precision_diffusion,
|
||||
n10.cd_nomenclature AS stade_vie,
|
||||
n11.cd_nomenclature AS sexe,
|
||||
n12.cd_nomenclature AS objet_denombrement,
|
||||
n13.cd_nomenclature AS type_denombrement,
|
||||
n14.cd_nomenclature AS niveau_sensibilite,
|
||||
n15.cd_nomenclature AS statut_observation,
|
||||
n16.cd_nomenclature AS floutage_dee,
|
||||
n17.cd_nomenclature AS statut_source,
|
||||
n18.cd_nomenclature AS type_info_geo,
|
||||
n19.cd_nomenclature AS methode_determination,
|
||||
n20.cd_nomenclature AS statut_validation,
|
||||
coalesce(s.meta_update_date, s.meta_create_date) AS derniere_action
|
||||
FROM gn_synthese.synthese s
|
||||
JOIN ds ON ds.id_dataset = s.id_dataset
|
||||
JOIN af ON ds.id_acquisition_framework = af.id_acquisition_framework
|
||||
LEFT JOIN ref_habitats.habref h ON h.cd_hab = s.cd_hab
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n1 ON s.id_nomenclature_geo_object_nature = n1.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n2 ON s.id_nomenclature_grp_typ = n2.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n3 ON s.id_nomenclature_behaviour = n3.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n4 ON s.id_nomenclature_obs_technique = n4.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n5 ON s.id_nomenclature_bio_status = n5.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n6 ON s.id_nomenclature_bio_condition = n6.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n7 ON s.id_nomenclature_naturalness = n7.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n8 ON s.id_nomenclature_exist_proof = n8.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n9 ON s.id_nomenclature_diffusion_level = n9.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n10 ON s.id_nomenclature_life_stage = n10.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n11 ON s.id_nomenclature_sex = n11.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n12 ON s.id_nomenclature_obj_count = n12.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n13 ON s.id_nomenclature_type_count = n13.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n14 ON s.id_nomenclature_sensitivity = n14.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n15 ON s.id_nomenclature_observation_status = n15.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n16 ON s.id_nomenclature_blurring = n16.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n17 ON s.id_nomenclature_source_status = n17.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n18 ON s.id_nomenclature_info_geo_type = n18.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n19 ON s.id_nomenclature_determination_method = n19.id_nomenclature
|
||||
LEFT JOIN ref_nomenclatures.t_nomenclatures n20 ON s.id_nomenclature_valid_status = n20.id_nomenclature
|
||||
|
||||
-- ADD CEN_38
|
||||
JOIN taxonomie.taxref t ON t.cd_nom = s.cd_nom
|
||||
JOIN gn_synthese.t_sources ts ON ts.id_source = s.id_source
|
||||
|
||||
WHERE t.regne = 'Animalia'
|
||||
AND t.phylum = 'Chordata'
|
||||
-- exclision CA ([SICEN] Echange données partenaires,[TEST] : migration sicen,)
|
||||
AND ds.id_acquisition_framework NOT IN (
|
||||
53,1,2,6, -- CA tests
|
||||
84, -- Echange de données PG Chravas
|
||||
85, -- Echange de données
|
||||
116 -- Echange de données issus de SICEN
|
||||
)
|
||||
-- exclision JDD `Observations opportunistes du CEN Isère importé depuis Faune Isère`
|
||||
AND ds.id_dataset NOT IN (185,377,236,409)
|
||||
-- exclision CA (`Gestion de l'Espace Naturel Sensible du Méandre des Oves`, `Gestion de la Réserve Naturelle Nationale de l'Ile de la Platière`, `Gestion des Natura 2000 FR 8201749 « Milieux alluviaux et aquatiques de l’Île de la Platière » et FR 8212012 « Île de la Platière »`, `RNN Platière`
|
||||
--AND (ts.id_module = 7 AND ds.id_acquisition_framework IN (7,38,39,42,44,45,46,48,55,57,58,60)
|
||||
AND (
|
||||
unaccent(observers) LIKE ANY (array['%%Conservatoire d%%Espaces Naturels Isere%%', '%%Conservatoire d%%Espaces Naturels d%%Isere%%', '%%CEN Isere%%', '%%Association des Amis de l%%Ile de la Platiere%%'])
|
||||
OR observers NOT LIKE '%%(%%)%%'
|
||||
)
|
||||
AND unaccent(observers) NOT LIKE ALL(array[
|
||||
'%%Sympetrum%%','%%Departement%%','%%Rhone%%','%%Oxalis%%',
|
||||
'%%LPO%%','%%GRPLS%%','%%Parvi%%','%%CD Isere%%','%%Personnel%%','%%Independant%%'
|
||||
'%%DELOCHE Denis%%'
|
||||
])
|
||||
AND observers NOT IN ('', 'Benoit Dodelin',', ')
|
||||
-- statut_validation = 'Certain - très probable'
|
||||
-- AND n20.cd_nomenclature = '1'
|
||||
ORDER BY s.id_synthese;
|
||||
'''
|
||||
|
||||
with con_gn.begin() as cnx:
|
||||
cnx.execute(sql)
|
||||
@@ -0,0 +1,19 @@
|
||||
# Evolution des configurations dans le schéma `pr_zh`
|
||||
## table `bib_organismes`
|
||||
- Augmentation des containtes data_type :
|
||||
abbreviation::varchar(9)
|
||||
## table `pr_zh`
|
||||
- Possibilité d'enregistrer plusieurs auteurs :
|
||||
create_author::json
|
||||
update_author::json
|
||||
|
||||
## table `cor_impact_types`
|
||||
- Les relations [`id_impact` vs `id_impact_type`] implémentées à l'installation du module homètent les relations homologues
|
||||
Ajout des relations manquantes avec un statut inactif
|
||||
exemple : [1012 vs 1250] == ['20 - Pollutions et nuisances' vs '20- POLLUTIONS ET NUISANCES']
|
||||
|
||||
## table `bib_cb`
|
||||
- Incompréhension sur la liste recensée qui semble incomplète et qui considère une liste CORINE BIOTOPE restrainte.
|
||||
|
||||
# Normalisation du référenciel Nomenclature
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
from pycen import con_gn, update_to_sql
|
||||
import geopandas as gpd
|
||||
from geoalchemy2 import Geometry
|
||||
|
||||
sql = 'SELECT id_zh, geom FROM pr_zh.t_zh'
|
||||
df = gpd.read_postgis(sql, con_gn)
|
||||
df.geom = df.buffer(0)
|
||||
|
||||
update_to_sql(
|
||||
# gpd.pd.DataFrame(df),
|
||||
df,
|
||||
con_gn,
|
||||
't_zh',
|
||||
'pr_zh',
|
||||
'id_zh',
|
||||
dtype={
|
||||
'geometry': Geometry(srid=4326)
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,773 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-.
|
||||
|
||||
import geopandas as gpd
|
||||
from uuid import uuid4
|
||||
import json,urllib.request
|
||||
from numpy import ndarray
|
||||
from geoalchemy2 import Geometry
|
||||
from sqlalchemy import text
|
||||
|
||||
dict_columns = {
|
||||
'code_zh': 'code',
|
||||
'nom_zh': 'main_name',
|
||||
'date_visite':'create_date',
|
||||
'remark_activity': 'global_remark_activity'
|
||||
}
|
||||
|
||||
dict_type_milieux = {
|
||||
'FU':'Fourré humide',
|
||||
'BFH':'Boisement feuillu humide',
|
||||
'BCH':'Boisement de conifères humide',
|
||||
'AL':'Végétation herbacée pionnière des alluvions',
|
||||
'GH':'Communauté de grands hélophytes',
|
||||
'RB':'Communauté de petits hélophytes',
|
||||
'MC':'Magnocariçaie',
|
||||
'BM':'Bas-marais et marais de transition',
|
||||
'MG':'Mégaphorbiaie',
|
||||
'AQ':'Végétation aquatique',
|
||||
'FO':'Végétation fontinale (sources, mouillères)',
|
||||
'EC':"Végétation amphibie des bordure d'eaux courantes",
|
||||
'PH':'Prairie humide (et pelouse humide)',
|
||||
}
|
||||
|
||||
|
||||
def normalize_connexion(con):
|
||||
if gpd.pd.__version__>'2.1.2':
|
||||
return con.raw_connection()
|
||||
else:
|
||||
return con
|
||||
|
||||
|
||||
def get_cor_impact_types(con):
|
||||
res = gpd.pd.read_sql_table('cor_impact_types',con,'pr_zh')
|
||||
return dict(zip(res.id_impact,res.id_cor_impact_types))
|
||||
|
||||
|
||||
def get_nomenclature_id(con,cd,typ):
|
||||
"""
|
||||
Get the id_nomenclature for a given cd and typ.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
con : sqlalchemy.engine.Engine
|
||||
The database connection.
|
||||
cd : str
|
||||
The code of the nomenclature.
|
||||
typ : str
|
||||
The type of the nomenclature.
|
||||
|
||||
Returns
|
||||
-------
|
||||
int or None
|
||||
The id of the nomenclature or None if the nomenclature does not exist.
|
||||
"""
|
||||
sql = """
|
||||
SELECT ref_nomenclatures.get_id_nomenclature('{typ}', '{cd}')
|
||||
""".format(cd=cd,typ=typ)
|
||||
with con.begin() as cnx:
|
||||
res = cnx.execute(text(sql))
|
||||
|
||||
return res.one()[0] if res else None
|
||||
|
||||
|
||||
def _cor_zh_area(tzh_code,con,typ,cover=False):
|
||||
"""
|
||||
@tzh : pd.Serie. Série de valeurs
|
||||
correspondants à la colonne pr_zh.t_zh."code".
|
||||
@typ : str. COM, DEP, ref_geo
|
||||
"""
|
||||
from math import ceil
|
||||
|
||||
table = 'cor_zh_area'
|
||||
|
||||
sqltzh = """
|
||||
SELECT zh.id_zh, zh.geom FROM pr_zh.t_zh zh WHERE zh."code" in {tzh_code}
|
||||
""".format(tzh_code=tuple(tzh_code))
|
||||
tzh = gpd.read_postgis(sqltzh,con,crs=4326)
|
||||
|
||||
if tzh.crs.srs.lower()=='epsg:4326':
|
||||
tzh.to_crs(2154,inplace=True)
|
||||
|
||||
sqllarea = """
|
||||
SELECT l.id_area, l.geom FROM ref_geo.l_areas l
|
||||
JOIN ref_geo.bib_areas_types bib USING (id_type)
|
||||
WHERE bib.type_code='{typ}' and l."enable"
|
||||
""".format(typ=typ)
|
||||
larea = gpd.read_postgis(sqllarea,con,crs=2154)
|
||||
|
||||
df = _calc_recouvrmt(larea,tzh).rename(columns={'perc_rcvmt':'cover'})
|
||||
|
||||
if cover:
|
||||
df['cover'] = [ceil(x) for x in df.cover]
|
||||
else :
|
||||
df.drop(columns=['cover'],inplace=True)
|
||||
|
||||
# return df
|
||||
if not df.empty:
|
||||
df.to_sql(
|
||||
name=table, con=con, schema='pr_zh',
|
||||
if_exists='append', index=False
|
||||
)
|
||||
print('INSERT %i correspondances'%df.shape[0])
|
||||
else:
|
||||
print('AUCUNE nouvelles correspondances identifiées')
|
||||
|
||||
|
||||
def _calc_recouvrmt(df1,df2,how='inner'):
|
||||
'''
|
||||
Calcule le recouvrement de df2 sur df1
|
||||
pour chaque géométrie de df1:
|
||||
Parameters
|
||||
----------
|
||||
df1 : GeoDataFrame.
|
||||
df2 : GeoDataFrame.
|
||||
'''
|
||||
iddf1 = df1.columns[0]
|
||||
iddf2 = df2.columns[0]
|
||||
|
||||
# Jointure spaciale
|
||||
tmp = gpd.sjoin(
|
||||
df1,
|
||||
df2[['geom']],
|
||||
predicate = 'intersects',
|
||||
how = how)
|
||||
|
||||
tmp.dropna(subset=['index_right'],inplace=True)
|
||||
tmp.index_right = tmp.index_right.astype(int)
|
||||
tmp.reset_index(inplace=True)
|
||||
|
||||
tmp = tmp.join(
|
||||
df2[['geom',iddf2]].rename(columns={'geom': 'right_geom'}),
|
||||
on=['index_right'], how='left')
|
||||
|
||||
tmp2 = tmp[['index_right','right_geom',iddf2]].copy() \
|
||||
.rename(columns={'right_geom': 'geom'}) \
|
||||
.set_geometry('geom')
|
||||
|
||||
tmp1 = tmp[[iddf1,'geom']].copy() \
|
||||
.set_geometry('geom')
|
||||
|
||||
if not tmp1.geom.values.is_valid.all():
|
||||
tmp1.loc[~tmp1.geom.values.is_valid,'geom'] = tmp1.loc[~tmp1.geom.values.is_valid,'geom'].buffer(0)
|
||||
|
||||
if not tmp2.geom.values.is_valid.all():
|
||||
tmp2.loc[~tmp2.geom.values.is_valid,'geom'] = tmp2.loc[~tmp2.geom.values.is_valid,'geom'].buffer(0)
|
||||
|
||||
tmp['perc_rcvmt'] = (tmp1.intersection(tmp2).area/tmp1.area)*100
|
||||
|
||||
return tmp[[iddf1,iddf2,'perc_rcvmt']]
|
||||
|
||||
|
||||
def _cor_zh_(tzh_code,con,typ):
|
||||
"""
|
||||
@tzh : pd.Serie. Série de valeurs
|
||||
correspondants à la colonne pr_zh.t_zh."code".
|
||||
@typ : str. [hydro,rb]
|
||||
"""
|
||||
typ = typ.lower()
|
||||
table = 'cor_zh_%s'%typ
|
||||
tab_typ = 't_hydro_area' if typ == 'hydro' else 't_river_basin'
|
||||
id_typ = 'id_hydro' if typ == 'hydro' else 'id_rb'
|
||||
sql = '''
|
||||
SELECT h.{id_typ},zh.id_zh
|
||||
FROM pr_zh.{tab_typ} h, pr_zh.t_zh zh
|
||||
WHERE zh."code" in {tzh_code}
|
||||
AND ST_INTERSECTS( ST_SetSRID(h.geom,4326),ST_MakeValid(ST_SetSRID(zh.geom,4326)))
|
||||
AND (h.{id_typ},zh.id_zh) NOT IN (SELECT {id_typ},id_zh FROM pr_zh.{tab_to})
|
||||
;'''.format(
|
||||
tzh_code = tuple(tzh_code),
|
||||
id_typ = id_typ,
|
||||
tab_typ = tab_typ,
|
||||
tab_to = table)
|
||||
df = gpd.pd.read_sql_query(sql,con)
|
||||
|
||||
if not df.empty:
|
||||
df.to_sql(
|
||||
name=table, con=con, schema='pr_zh',
|
||||
if_exists='append', index=False
|
||||
)
|
||||
print('INSERT %i correspondances'%df.shape[0])
|
||||
else:
|
||||
print('AUCUNE nouvelles correspondances identifiées')
|
||||
|
||||
|
||||
|
||||
def to_tzh(df,con,lst_area):
|
||||
tab = 't_zh'
|
||||
sch = 'pr_zh'
|
||||
con.connect()
|
||||
lst_columns = [
|
||||
x['name'] for x in con.dialect.get_columns(con.connect(),tab,sch) if x['name'] in df.columns
|
||||
]
|
||||
|
||||
# con = normalize_connexion(con)
|
||||
to_ins = df[lst_columns].copy()
|
||||
|
||||
lst = []
|
||||
for r,row in to_ins.iterrows():
|
||||
tmplst = []
|
||||
for v, vals in row.items():
|
||||
if isinstance(vals,(str,int)):
|
||||
if isinstance(vals,str) and vals.__contains__("'"):
|
||||
vals = vals.replace("'","''")
|
||||
tmplst.append(vals)
|
||||
else:
|
||||
tmplst.append(str(vals))
|
||||
lst.append(str(tuple(tmplst)))
|
||||
|
||||
sqlValue = ','.join(lst).replace('"',"'").replace("'nan'",'NULL')
|
||||
sql = '''
|
||||
INSERT INTO pr_zh.t_zh ({cols}) VALUES {vals}
|
||||
RETURNING id_zh,code;
|
||||
'''.format(
|
||||
cols=','.join(to_ins.columns),
|
||||
vals=sqlValue
|
||||
)
|
||||
|
||||
|
||||
try:
|
||||
with con.begin() as cnx:
|
||||
_res = cnx.execute(text(sql))
|
||||
res = _res.fetchall()
|
||||
# to_ins.to_postgis(tab,con,sch,if_exists='append',index=False,)
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
res_df = gpd.pd.DataFrame(res)
|
||||
for area in lst_area:
|
||||
cov = True if area == 'COM' else False
|
||||
_cor_zh_area(tzh_code=res_df.code,con=con,typ=area,cover=cov)
|
||||
print('INSERT cor_zh_area %s OK !'%area)
|
||||
|
||||
_cor_zh_(tzh_code=res_df.code,con=con,typ='hydro')
|
||||
print('INSERT cor_zh_hydro OK !')
|
||||
_cor_zh_(tzh_code=res_df.code,con=con,typ='rb')
|
||||
print('INSERT cor_zh_rb OK !')
|
||||
|
||||
print("INSERT data TO t_zh OK !")
|
||||
return res_df
|
||||
# return gpd.pd.read_sql(
|
||||
# 'SELECT id_zh,code FROM pr_zh.t_zh WHERE code {sym} {lst};'.format(
|
||||
# sym='IN' if to_ins.shape[0] > 1 else '=',
|
||||
# lst=str(tuple(to_ins.code)) if to_ins.shape[0] > 1 else "'%s'"%to_ins.code.to_list()[0]
|
||||
# ),
|
||||
# con=con)
|
||||
# id_zh = df[['code','pk']].rename(columns={'pk':'id_zh'})
|
||||
|
||||
def to_cor_lim_list(cor_lim,con):
|
||||
"""
|
||||
Insert into cor_lim_list the list of delimitation for each zh area
|
||||
:param cor_lim: a dataframe with columns id_lim_list,cd_nomenclature_delimitation
|
||||
:param con: a sqlalchemy connection
|
||||
:return: None
|
||||
"""
|
||||
typ_class = list(set([tup.cd_nomenclature_delimitation.__class__ for tup in cor_lim.itertuples()]))
|
||||
if typ_class[0] == list:
|
||||
_cor_lim = (cor_lim
|
||||
.set_index('id_lim_list')
|
||||
.cd_nomenclature_delimitation
|
||||
.explode()
|
||||
.to_frame('cd_lim'))
|
||||
else:
|
||||
_cor_lim = (cor_lim
|
||||
.set_index('id_lim_list')
|
||||
.cd_nomenclature_delimitation.str.split(',',expand=True)
|
||||
.stack()
|
||||
.str.strip()
|
||||
.to_frame('cd_lim')
|
||||
.droplevel(-1)
|
||||
)
|
||||
_cor_lim['id_lim'] = [get_nomenclature_id(con,x,'CRIT_DELIM') for x in _cor_lim.cd_lim]
|
||||
_cor_lim.drop('cd_lim',axis=1,inplace=True)
|
||||
|
||||
con = normalize_connexion(con)
|
||||
try:
|
||||
_cor_lim.to_sql(
|
||||
name='cor_lim_list',
|
||||
con=con,
|
||||
schema='pr_zh',
|
||||
index=True,
|
||||
if_exists='append',
|
||||
method='multi'
|
||||
)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
finally:
|
||||
print("INSERT Délimitation TO cor_lim_list OK !")
|
||||
|
||||
def to_cor_zh_cb(id_zh,df,con):
|
||||
"""
|
||||
Insert habitats to cor_zh_cb table.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
id_zh : pd.DataFrame
|
||||
DataFrame containing 'code' and 'id_zh' columns.
|
||||
df : pd.DataFrame
|
||||
DataFrame containing 'code' and 'habitat_corine_biotope' columns.
|
||||
con : sqlalchemy.engine.Engine
|
||||
Connection to the geonature database.
|
||||
|
||||
Returns
|
||||
-------
|
||||
None
|
||||
"""
|
||||
|
||||
cor_zh_cb = (df
|
||||
.merge(id_zh,how='left',on='code')
|
||||
.set_index('id_zh')
|
||||
.habitat_corine_biotope.str.split(',')
|
||||
.explode()
|
||||
.str.strip()
|
||||
.to_frame('lb_code')
|
||||
)
|
||||
|
||||
con = normalize_connexion(con)
|
||||
try:
|
||||
cor_zh_cb.to_sql(
|
||||
name='cor_zh_cb',
|
||||
con=con,
|
||||
schema='pr_zh',
|
||||
index=True,
|
||||
if_exists='append',
|
||||
method='multi'
|
||||
)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
finally:
|
||||
print("INSERT habitats to cor_zh_cb OK !")
|
||||
|
||||
|
||||
def check_cd_nomenclature_impact(cd):
|
||||
dizaine = range(0,100,10)
|
||||
_cd = [x+'.0' if float(x) not in dizaine and not x.endswith('.0') else x for x in cd]
|
||||
return _cd
|
||||
|
||||
|
||||
def to_t_activity(id_zh,actv,con):
|
||||
_activ = (actv
|
||||
.merge(id_zh,how='left',on='code')
|
||||
.drop('code',axis=1))
|
||||
|
||||
t_activ = gpd.pd.DataFrame()
|
||||
for i,x in _activ.iterrows():
|
||||
if isinstance(x['acti_impact'],str):
|
||||
res = gpd.pd.DataFrame(json.loads(x['acti_impact']))
|
||||
else :
|
||||
res = gpd.pd.DataFrame(x['acti_impact'])
|
||||
res['id_zh'] = x['id_zh']
|
||||
t_activ = gpd.pd.concat([t_activ,res])
|
||||
t_activ.set_index('id_zh',inplace=True)
|
||||
t_activ.dropna(inplace=True,axis=1,how='all')
|
||||
t_activ.dropna(inplace=True,axis=0,how='all')
|
||||
|
||||
t_activ['id_activity'] = [get_nomenclature_id(con,x,'ACTIV_HUM') for x in t_activ.cd_activite_humaine]
|
||||
t_activ['id_position'] = [get_nomenclature_id(con,x,'LOCALISATION') for x in t_activ.localisation]
|
||||
t_activ['id_impact_list'] = [uuid4() for _ in range(len(t_activ))]
|
||||
|
||||
impact_list = (t_activ[['id_impact_list','impact']]
|
||||
.set_index('id_impact_list')
|
||||
.impact.explode()
|
||||
.to_frame('cd_impact')
|
||||
.reset_index()
|
||||
)
|
||||
impact_list['_cd_impact'] = check_cd_nomenclature_impact(impact_list.cd_impact)
|
||||
impact_list['id_cor_impact_types'] = [get_nomenclature_id(con,x,'IMPACTS') for x in impact_list._cd_impact]
|
||||
|
||||
cor_imp_typ = get_cor_impact_types(con)
|
||||
impact_list.replace({'id_cor_impact_types':cor_imp_typ}, inplace=True)
|
||||
|
||||
t_activ.drop(['cd_activite_humaine','localisation','impact'],axis=1,inplace=True)
|
||||
impact_list.drop(['cd_impact','_cd_impact'],axis=1,inplace=True)
|
||||
|
||||
con = normalize_connexion(con)
|
||||
try:
|
||||
t_activ.to_sql(
|
||||
name='t_activity',
|
||||
con=con,
|
||||
schema='pr_zh',
|
||||
index=True,
|
||||
if_exists='append',
|
||||
method='multi'
|
||||
)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
finally:
|
||||
print("INSERT activity to t_activity OK !")
|
||||
|
||||
try:
|
||||
impact_list.to_sql(
|
||||
name='cor_impact_list',
|
||||
con=con,
|
||||
schema='pr_zh',
|
||||
index=False,
|
||||
if_exists='append',
|
||||
method='multi'
|
||||
)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
finally:
|
||||
print("INSERT impact to cor_impact_list OK !")
|
||||
|
||||
|
||||
def check_habitat(habitat,con):
|
||||
sql = '''
|
||||
SELECT lb_code FROM pr_zh.bib_cb
|
||||
WHERE lb_code {symbol} {code}
|
||||
'''.format(
|
||||
symbol='IN' if isinstance(habitat,(list,gpd.pd.Series,ndarray)) else '=',
|
||||
code=tuple(habitat) if isinstance(habitat,(list,gpd.pd.Series,ndarray)) else f"'{habitat}'"
|
||||
)
|
||||
with con.begin() as cnx:
|
||||
res = cnx.execute(text(sql)).all()
|
||||
return [x[0] for x in res if x]
|
||||
|
||||
|
||||
def filter_habitat(habitat,con):
|
||||
iscd_zh = check_habitat(habitat,con)
|
||||
_cd_zh = habitat[habitat.isin(iscd_zh)]
|
||||
_cd_notzh = habitat[~habitat.isin(iscd_zh)]
|
||||
cd_zh = _cd_zh.groupby(_cd_zh.index).aggregate(', '.join)
|
||||
cd_notzh = _cd_notzh.groupby(_cd_notzh.index).aggregate(', '.join)
|
||||
return cd_zh,cd_notzh
|
||||
|
||||
|
||||
def check_observ(obs,id_org,con):
|
||||
_obs = normalize_observers(obs)
|
||||
con = normalize_connexion(con)
|
||||
sql = '''
|
||||
SELECT
|
||||
r.id_role AS ids_observers,
|
||||
CONCAT(r.nom_role, ' ', prenom_role) AS observers
|
||||
FROM utilisateurs.t_roles r
|
||||
JOIN utilisateurs.bib_organismes USING (id_organisme)
|
||||
WHERE CONCAT(UPPER(r.nom_role), ' ', INITCAP(prenom_role)) {symbol} {code}
|
||||
AND id_organisme = '{org}'
|
||||
'''.format(
|
||||
symbol='IN' if isinstance(_obs,(list,gpd.pd.Series,ndarray)) else '=',
|
||||
code=tuple(_obs) if isinstance(_obs,(list,gpd.pd.Series,ndarray)) else f"'{_obs}'",
|
||||
org=id_org
|
||||
)
|
||||
res = (gpd.pd.read_sql(
|
||||
sql,
|
||||
con)
|
||||
.merge(_obs.to_frame('observers'),how='right',on='observers'))
|
||||
return res
|
||||
|
||||
def insert_observ(obs,id_org,con):
|
||||
check = check_observ(obs,id_org,con)
|
||||
usr_missing = check[check.ids_observers.isnull()].drop_duplicates()
|
||||
if usr_missing.empty:
|
||||
print("Observateurs trouvés dans le schéma `utilisateurs`")
|
||||
return check
|
||||
Q = input("Observateurs `{}` non trouvés dans le schéma `utilisateurs`, voulez-vous les ajouter ? (y/n) ".format(usr_missing.observers.to_list()))
|
||||
if Q.lower() == 'y':
|
||||
id_obs_miss = _insert_observ(usr_missing,id_org,con)
|
||||
for i,row in id_obs_miss.iterrows():
|
||||
check.loc[check.observers==row.observers,'ids_observers'] = row.id_role
|
||||
# check = check_observ(obs,id_org,con)
|
||||
return check.astype({'ids_observers':int})
|
||||
else:
|
||||
return 'No'
|
||||
|
||||
def _insert_observ(obs,id_org,con):
|
||||
_obs = gpd.pd.concat(
|
||||
[obs,obs.observers.str.split(' ',n=-1,expand=True)],
|
||||
axis=1,ignore_index=True)
|
||||
_obs['id_org'] = id_org
|
||||
|
||||
sql ='''
|
||||
INSERT INTO utilisateurs.t_roles (nom_role,prenom_role,id_organisme)
|
||||
VALUES {vals}
|
||||
--ON CONFLICT (nom_organisme) DO NOTHING
|
||||
RETURNING id_role, CONCAT(UPPER(nom_role), ' ', INITCAP(prenom_role)) AS observers;
|
||||
'''.format(vals=','.join(["('%s','%s',%i)"%(row[3],row[4],row[5]) for row in _obs.itertuples()]))
|
||||
|
||||
# con = normalize_connexion(con)
|
||||
# res = gpd.pd.read_sql(sql,con)
|
||||
# return res
|
||||
with con.begin() as cnx:
|
||||
_res = cnx.execute(text(sql))
|
||||
res = _res.fetchall()
|
||||
return gpd.pd.DataFrame(res)
|
||||
|
||||
def normalize_observers(obs):
|
||||
_obs = obs.str.split(' ',expand=True)
|
||||
_obs[0] = _obs[0].str.upper()
|
||||
_obs[1] = _obs[1].str.title()
|
||||
_obs_stack = _obs.stack().droplevel(-1)
|
||||
return _obs_stack.groupby(_obs_stack.index).aggregate(' '.join)
|
||||
|
||||
|
||||
def select_orga_user(org,con):
|
||||
sql ='''
|
||||
SELECT id_organisme FROM utilisateurs.bib_organismes
|
||||
WHERE nom_organisme = '{nom}'
|
||||
'''.format(nom=org[0])
|
||||
with con.begin() as cnx:
|
||||
_res = cnx.execute(text(sql))
|
||||
res = _res.one_or_none()
|
||||
return res
|
||||
|
||||
|
||||
def insert_orga_user(org,con):
|
||||
res = select_orga_user(org,con)
|
||||
if not res :
|
||||
Q = input("Organisme `{}` non trouvé dans le schéma `utilisateurs`, voulez-vous l'ajouter ? (y/n) ".format(org[0]))
|
||||
if Q.lower() != 'y':
|
||||
return 'No'
|
||||
sql ='''
|
||||
INSERT INTO utilisateurs.bib_organismes (nom_organisme)
|
||||
VALUES ('{nom}')
|
||||
--ON CONFLICT (nom_organisme) DO NOTHING
|
||||
RETURNING id_organisme;
|
||||
'''.format(nom=org[0])
|
||||
with con.begin() as cnx:
|
||||
_res = cnx.execute(text(sql))
|
||||
res = _res.one_or_none()
|
||||
else:
|
||||
print("Organisme `{}` existant dans le schéma `utilisateurs`".format(org[0]))
|
||||
return res[0]
|
||||
|
||||
def select_orga_przh(org,con):
|
||||
sql ='''
|
||||
SELECT id_org FROM pr_zh.bib_organismes
|
||||
WHERE name = '{nom}';
|
||||
'''.format(nom=org[0])
|
||||
with con.begin() as cnx:
|
||||
_res = cnx.execute(text(sql))
|
||||
res = _res.one_or_none()
|
||||
return res
|
||||
|
||||
def insert_orga_przh(org,con):
|
||||
res = select_orga_przh(org,con)
|
||||
if not res :
|
||||
Q = input("Organisme `{}` non trouvé dans le schéma `pr_zh`, voulez-vous l'ajouter ? (y/n) ".format(org[0]))
|
||||
if Q.lower() != 'y':
|
||||
return 'No'
|
||||
sql ='''
|
||||
INSERT INTO pr_zh.bib_organismes (name,abbrevation,is_op_org)
|
||||
VALUES ('{nom}', {abbrev}, TRUE)
|
||||
--ON CONFLICT (name,abbrevation) DO NOTHING
|
||||
RETURNING id_org;
|
||||
'''.format(nom=org[0],abbrev=f"'{org[1]}'" if org[1] else 'NULL')
|
||||
with con.begin() as cnx:
|
||||
_res = cnx.execute(text(sql))
|
||||
res = _res.fetchall()
|
||||
else:
|
||||
print("Organisme `{}` existant dans le schéma `pr_zh`".format(org[0]))
|
||||
return res[0]
|
||||
|
||||
def insert_users_missing(user,org,con):
|
||||
id_org_przh = insert_orga_przh(org,con)
|
||||
id_user_orga = insert_orga_user(org,con)
|
||||
obsv = insert_observ(user,id_user_orga,con)
|
||||
return obsv, id_org_przh
|
||||
|
||||
def filter_zh(code,con):
|
||||
sql ='''
|
||||
SELECT code FROM pr_zh.t_zh
|
||||
WHERE code IN {}
|
||||
'''.format(code)
|
||||
|
||||
con = normalize_connexion(con)
|
||||
res = gpd.pd.read_sql(sql,con)
|
||||
return res
|
||||
|
||||
def insert_zh_fromapi(
|
||||
url,con,dep_filter,orga,lst_area=['DEP','COM'],
|
||||
prefix_hab_rq='',code_rename={}):
|
||||
"""
|
||||
Insert data from API into geonature database
|
||||
|
||||
Parameters
|
||||
----------
|
||||
url : str
|
||||
url of the API
|
||||
con : sqlalchemy.engine.Engine
|
||||
connection to the geonature database
|
||||
dep_filter : str
|
||||
filter code_zh by this department code
|
||||
prefix_hab_rq : str
|
||||
prefix to add to remark_pres field for habitats not in geonature database
|
||||
code_rename : dict
|
||||
Dictionnary of zh code
|
||||
|
||||
|
||||
Returns
|
||||
-------
|
||||
None
|
||||
"""
|
||||
|
||||
api = gpd.read_file(url, engine='fiona')
|
||||
if api.empty:
|
||||
print("Aucune zone humide trouvée")
|
||||
print("Arrêt du script !")
|
||||
return None
|
||||
|
||||
api.rename(columns=dict_columns, inplace=True)
|
||||
if code_rename :
|
||||
api.replace({'code':code_rename},inplace=True)
|
||||
|
||||
zh_filter = filter_zh(
|
||||
tuple(api.code.dropna()),
|
||||
con
|
||||
)
|
||||
|
||||
nzh_tobdd = api[
|
||||
(api.action=='Créer')&
|
||||
(~api.code.isin(zh_filter.code.tolist()))&
|
||||
(~api.cd_typo_sdage.isna())
|
||||
].shape[0]
|
||||
print("L'api contient %s zones .."%api.shape[0])
|
||||
print("%s zones sont taguée `Problème` .."%api[api.action=='Problème'].shape[0])
|
||||
print("%s zones dont la typologie SDAGE est inconnue et sont écartées"%api[api.cd_typo_sdage.isna()].shape[0])
|
||||
print("%s zones taguée `Créer` existent déjà .."%zh_filter.shape[0])
|
||||
print("%s zones taguée `Créer` sont à intégrer .."%nzh_tobdd)
|
||||
|
||||
if nzh_tobdd == 0 :
|
||||
print("\nAucune nouvelles zones humides sont à intégrée ..")
|
||||
print("Arrêt du script !")
|
||||
return None
|
||||
|
||||
Q = input('Souhaitez-vous continuer ? (Y/n) : ')
|
||||
if Q.lower() != 'y':
|
||||
print('Arrêt du code ..')
|
||||
return None
|
||||
|
||||
df = (api[
|
||||
(api.action=='Créer')&
|
||||
(~api.code.isin(zh_filter.code.tolist()))&
|
||||
(~api.cd_typo_sdage.isna())
|
||||
].rename_geometry('geom')
|
||||
# .merge(load_missing_propertie(url,'cd_nomenclature_delimitation',dep_filter), on='pk')
|
||||
)
|
||||
|
||||
df.observateur = normalize_observers(df.observateur)
|
||||
users, id_org = insert_users_missing(df.observateur,orga,con)
|
||||
df['id_org'] = id_org
|
||||
df = (df
|
||||
.merge(users.drop_duplicates(),how='left',left_on='observateur',right_on='observers')
|
||||
.rename(columns={'ids_observers':'id_role'})
|
||||
)
|
||||
|
||||
df['zh_uuid'] = [uuid4() for _ in range(len(df))]
|
||||
df['id_lim_list'] = [uuid4() for _ in range(len(df))]
|
||||
df['id_sdage'] = [get_nomenclature_id(con,x,'SDAGE') for x in df.cd_typo_sdage]
|
||||
df['create_author'] = df.id_role.copy()
|
||||
df['update_author'] = df.id_role.copy()
|
||||
df['area'] = df.to_crs(2154).area/10000
|
||||
|
||||
cd_hab = (df
|
||||
.set_index('code')
|
||||
.habitat_corine_biotope
|
||||
.str.split(',').explode()
|
||||
.str.strip())
|
||||
|
||||
hab_zh,hab_notzh = filter_habitat(cd_hab,con)
|
||||
_df = (df
|
||||
.drop('habitat_corine_biotope',axis=1)
|
||||
.merge(hab_zh,how='left',right_index=True,left_on='code')
|
||||
.sort_values('code'))
|
||||
|
||||
# Si pas de champs remark_pres dans le tableau
|
||||
if 'remark_pres' not in _df.columns:
|
||||
_df = _df.merge(
|
||||
prefix_hab_rq + hab_notzh.rename('remark_pres'),
|
||||
how='left',right_index=True,left_on='code'
|
||||
)
|
||||
else :
|
||||
_df.remark_pres = (_df.remark_pres
|
||||
.str.split('_',expand=True)[0]
|
||||
.replace(dict_type_milieux)
|
||||
)
|
||||
_df.set_index('code',inplace=True)
|
||||
_df.loc[_df.index.isin(hab_notzh.index),'remark_pres'] = (
|
||||
_df[_df.index.isin(hab_notzh.index.tolist())]
|
||||
.remark_pres
|
||||
+ '\n' + prefix_hab_rq + hab_notzh
|
||||
)
|
||||
_df.reset_index(drop=False,inplace=True)
|
||||
|
||||
c = _df[_df.action=="Créer"].copy()
|
||||
u = _df[_df.action=="Modifier"].copy()
|
||||
|
||||
if not c.empty:
|
||||
# retrun c
|
||||
id_zh = to_tzh(c,con,lst_area)
|
||||
to_cor_zh_cb(id_zh,c,con)
|
||||
to_cor_lim_list(c[['id_lim_list','cd_nomenclature_delimitation']],con)
|
||||
to_t_activity(id_zh,c[['code','acti_impact']],con)
|
||||
if not u.empty:
|
||||
raise('Script à coder !')
|
||||
|
||||
|
||||
def load_missing_propertie(url,propertie,dep_filter='38'):
|
||||
"""
|
||||
Load from api the missing properties for the given url, propertie and dep_filter.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
url : str
|
||||
url of the api
|
||||
propertie : str
|
||||
name of the propertie to retrieve
|
||||
dep_filter : str
|
||||
filter for the department, default is '38'
|
||||
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
DataFrame with columns 'code' and propertie
|
||||
"""
|
||||
|
||||
data = urllib.request.urlopen(url).read()
|
||||
output = json.loads(data)
|
||||
features = output['items']['features']
|
||||
|
||||
res = {
|
||||
'pk':[x['properties']['pk'] for x in features if x['properties']['departement'] == dep_filter],
|
||||
propertie:[
|
||||
','.join(x['properties'][propertie])
|
||||
for x in features
|
||||
if x['properties']['departement'] == dep_filter],
|
||||
}
|
||||
return gpd.pd.DataFrame(res)
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.engine import URL
|
||||
# Parametres bdd
|
||||
user = 'xxxxxxx'
|
||||
pwd = "xxxxxxx"
|
||||
adr = 'xxxxxxx'
|
||||
base = 'xxxxxxx'
|
||||
url = URL.create("postgresql+psycopg2", username=user, password=pwd, host=adr, database=base)
|
||||
con_gn = create_engine(url)
|
||||
|
||||
# Numéro de département permettant d'identifier les zones humides concernées par le territoire
|
||||
# ['38', '05'], default : 38
|
||||
dep_filter = '05'
|
||||
# Préfixe ajouté dans le champs remark_pres lorsque des habitats décrits ne sont pas des habitats dit "humides" dans la listes des habitats humides définis en bdd (table bib_cb)
|
||||
prefix_hab_rmk = 'Autre(s) habitat(s) décrit(s) :\n'
|
||||
# [Nom de l'organisme, Abbreviation]
|
||||
organisme = ['Parc national des Écrins','PNE']
|
||||
|
||||
# Liste des type_area à intersecter pour remplir la table cor_zh_area
|
||||
listAreaCover = ['DEP','COM','APB','ZPS','SIC','ZNIEFF1','ZNIEFF2']
|
||||
|
||||
# Code zh de l'api à renomer si souhaité. Défaut = {}
|
||||
code_rename = {}
|
||||
|
||||
api = 'https://geonature.ecrins-parcnational.fr/api/exports/api/21?departement=%s'%dep_filter
|
||||
res = insert_zh_fromapi(
|
||||
url=api,
|
||||
con=con_gn,
|
||||
dep_filter=dep_filter,
|
||||
orga=organisme,
|
||||
lst_area=listAreaCover,
|
||||
prefix_hab_rq=prefix_hab_rmk,
|
||||
code_rename=code_rename
|
||||
)
|
||||
@@ -0,0 +1,22 @@
|
||||
from pycen import con_gn,update_to_sql
|
||||
from os import path
|
||||
import geopandas as gpd
|
||||
|
||||
PATH = '/media/cgeier/SRV/FICHIERS'
|
||||
FILE = 'OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/ETUDES/Zones humides/ACTUALISATION 2024/BE EPODE - LEZE/KIT INVENTAIRE38 MODIFIE/a envoyer prêt/GEOM/38CG0105_Epode2025 (à intégrer).gpkg'
|
||||
|
||||
tmp = (gpd.read_file(path.join(PATH, FILE))
|
||||
.rename(columns={'CODE_ZH':'code'})
|
||||
.rename_geometry('geom')
|
||||
)
|
||||
|
||||
if tmp.crs.srs.lower() != 'epsg:4326':
|
||||
tmp = tmp.to_crs(4326)
|
||||
|
||||
update_to_sql(
|
||||
tmp[['code','geom']],
|
||||
con_gn,
|
||||
't_zh',
|
||||
'pr_zh',
|
||||
'code'
|
||||
)
|
||||
@@ -0,0 +1,26 @@
|
||||
import geopandas as gpd
|
||||
from pycen import con_gn, update_to_sql
|
||||
from sqlalchemy import DateTime
|
||||
from os import path
|
||||
|
||||
PATH = '/home/colas/Documents/9_PROJETS/1_ZH/2024/CARTO/Couches SIG'
|
||||
FILE = 'ZH_zh DDT 2024_MJ_modif 2025.shp'
|
||||
|
||||
df = (gpd.read_file(path.join(PATH, FILE))
|
||||
.rename(columns={'Code ZH':'code','date':'update_date'})
|
||||
.rename_geometry('geom')
|
||||
)
|
||||
df.geom = df.geom.buffer(0).to_crs(4326)
|
||||
df['update_author'] = 24 # 24 = Mathieu Juton
|
||||
|
||||
|
||||
update_to_sql(
|
||||
df[['code','update_author', 'update_date','geom']],
|
||||
con_gn,
|
||||
't_zh',
|
||||
'pr_zh',
|
||||
'code',
|
||||
dtype={
|
||||
'update_date': DateTime
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
import geopandas as gpd
|
||||
from pycen import con_gn, update_to_sql
|
||||
from os import path
|
||||
|
||||
PATH = '/media/cgeier/SRV/FICHIERS'
|
||||
FILE = 'OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/ETUDES/Zones humides/diag_zh_GAM/ZH_Séchilienne/zh_sapey.gpkg'
|
||||
df = (gpd.read_file(path.join(PATH, FILE))
|
||||
.rename(columns={'code_zh':'code'})
|
||||
.rename_geometry('geom'))
|
||||
|
||||
if df.crs.srs.lower() != 'epsg:4326':
|
||||
df = df.to_crs(4326)
|
||||
|
||||
update_to_sql(
|
||||
df[['code','geom']],
|
||||
con_gn,
|
||||
't_zh',
|
||||
'pr_zh',
|
||||
'code'
|
||||
)
|
||||
|
||||
Executable
+1
@@ -0,0 +1 @@
|
||||
#!/usr/bin/env python3
|
||||
Executable
+263
@@ -0,0 +1,263 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
import geopandas as gpd
|
||||
from geoalchemy2 import Geometry
|
||||
|
||||
|
||||
def _calc_recouvrmt(df1,df2,how='inner'):
|
||||
'''
|
||||
Calcule le recouvrement de df2 sur df1
|
||||
pour chaque géométrie de df1:
|
||||
Parameters
|
||||
----------
|
||||
df1 : GeoDataFrame.
|
||||
df2 : GeoDataFrame.
|
||||
'''
|
||||
iddf1 = df1.columns[0]
|
||||
iddf2 = df2.columns[0]
|
||||
|
||||
# Jointure spaciale
|
||||
tmp = gpd.sjoin(
|
||||
df1,
|
||||
df2[['geom']],
|
||||
predicate = 'intersects',
|
||||
how = how)
|
||||
|
||||
tmp.dropna(subset=['index_right'],inplace=True)
|
||||
tmp.index_right = tmp.index_right.astype(int)
|
||||
tmp.reset_index(inplace=True)
|
||||
|
||||
tmp = tmp.join(
|
||||
df2[['geom',iddf2]].rename(columns={'geom': 'right_geom'}),
|
||||
on=['index_right'], how='left')
|
||||
|
||||
tmp2 = tmp[['index_right','right_geom',iddf2]].copy() \
|
||||
.rename(columns={'right_geom': 'geom'}) \
|
||||
.set_geometry('geom')
|
||||
|
||||
tmp1 = tmp[[iddf1,'geom']].copy() \
|
||||
.set_geometry('geom')
|
||||
|
||||
if not tmp1.geom.values.is_valid.all():
|
||||
tmp1.loc[~tmp1.geom.values.is_valid,'geom'] = tmp1.loc[~tmp1.geom.values.is_valid,'geom'].buffer(0)
|
||||
|
||||
if not tmp2.geom.values.is_valid.all():
|
||||
tmp2.loc[~tmp2.geom.values.is_valid,'geom'] = tmp2.loc[~tmp2.geom.values.is_valid,'geom'].buffer(0)
|
||||
|
||||
tmp['perc_rcvmt'] = (tmp1.intersection(tmp2).area/tmp1.area)*100
|
||||
|
||||
return tmp[[iddf1,iddf2,'perc_rcvmt']]
|
||||
|
||||
|
||||
def _cor_zh_area(con,tzh_code,typ,cover=False):
|
||||
"""
|
||||
@tzh : pd.Serie. Série de valeurs
|
||||
correspondants à la colonne pr_zh.t_zh."code".
|
||||
@typ : str. COM, DEP, ref_geo
|
||||
"""
|
||||
from math import ceil
|
||||
|
||||
table = 'cor_zh_area'
|
||||
|
||||
sqltzh = """
|
||||
SELECT zh.id_zh, zh.geom FROM pr_zh.t_zh zh WHERE zh."code" in {tzh_code}
|
||||
""".format(tzh_code=tuple(tzh_code))
|
||||
tzh = gpd.read_postgis(sqltzh,con,crs=4326)
|
||||
|
||||
if tzh.crs.srs=='epsg:4326':
|
||||
tzh.to_crs(2154,inplace=True)
|
||||
|
||||
if typ == 'COM':
|
||||
cd2 = ' AND l."enable";'
|
||||
else: cd2 = ';'
|
||||
|
||||
# sql = '''
|
||||
# SELECT l.id_area,l.geom,zh.id_zh
|
||||
# FROM ref_geo.l_areas l
|
||||
# JOIN ref_geo.bib_areas_types bib USING (id_type),
|
||||
# pr_zh.t_zh zh
|
||||
# WHERE zh."code" in {tzh_code}
|
||||
# AND bib.type_code='{typ}'
|
||||
# AND ST_INTERSECTS( ST_SetSRID(l.geom,2154), ST_MakeValid(ST_Transform(ST_SetSRID(zh.geom,4326),2154)) )
|
||||
# AND (l.id_area,zh.id_zh) NOT IN (SELECT id_area,id_zh FROM pr_zh.cor_zh_area)
|
||||
# {cd2}
|
||||
# '''.format(tzh_code=tuple(tzh_code),typ=typ,cd2=cd2)
|
||||
# # larea = gpd.pd.read_sql_query(sql,con)
|
||||
# larea = gpd.read_postgis(sql,con)
|
||||
sqllarea = """
|
||||
SELECT l.id_area, l.geom FROM ref_geo.l_areas l
|
||||
JOIN ref_geo.bib_areas_types bib USING (id_type)
|
||||
WHERE bib.type_code='{typ}' and l."enable"
|
||||
""".format(typ=typ)
|
||||
larea = gpd.read_postgis(sqllarea,con,crs=2154)
|
||||
|
||||
df = _calc_recouvrmt(larea,tzh).rename(columns={'perc_rcvmt':'cover'})
|
||||
|
||||
if cover:
|
||||
df['cover'] = [ceil(x) for x in df.cover]
|
||||
else :
|
||||
df.drop(columns=['cover'],inplace=True)
|
||||
|
||||
|
||||
if not df.empty:
|
||||
df.to_sql(
|
||||
name=table, con=con, schema='pr_zh',
|
||||
if_exists='append', index=False
|
||||
)
|
||||
print('INSERT %i correspondances'%df.shape[0])
|
||||
else:
|
||||
sql = '''
|
||||
SELECT l.id_area
|
||||
FROM ref_geo.l_areas l
|
||||
JOIN ref_geo.bib_areas_types bib USING (id_type)
|
||||
WHERE bib.type_code='{typ}'
|
||||
'''.format(typ=typ)
|
||||
res = gpd.pd.read_sql_query(sql,con)
|
||||
if not res.empty:
|
||||
print('AUCUNE nouvelles correspondances identifiées')
|
||||
else :
|
||||
print('AUCUNE geometrie dans la table `ref_geo.l_areas` pour le `type_code` %s'%typ)
|
||||
|
||||
|
||||
|
||||
def _cor_zh_(con,tzh_code,typ):
|
||||
"""
|
||||
@tzh : pd.Serie. Série de valeurs
|
||||
correspondants à la colonne pr_zh.t_zh."code".
|
||||
@typ : str. [hydro,rb]
|
||||
"""
|
||||
typ = typ.lower()
|
||||
table = 'cor_zh_%s'%typ
|
||||
tab_typ = 't_hydro_area' if typ == 'hydro' else 't_river_basin'
|
||||
id_typ = 'id_hydro' if typ == 'hydro' else 'id_rb'
|
||||
sql = '''
|
||||
SELECT h.{id_typ},zh.id_zh
|
||||
FROM pr_zh.{tab_typ} h, pr_zh.t_zh zh
|
||||
WHERE zh."code" in {tzh_code}
|
||||
AND ST_INTERSECTS( ST_SetSRID(h.geom,4326),ST_MakeValid(ST_SetSRID(zh.geom,4326)))
|
||||
AND (h.{id_typ},zh.id_zh) NOT IN (SELECT {id_typ},id_zh FROM pr_zh.{tab_to})
|
||||
;'''.format(
|
||||
tzh_code = tuple(tzh_code),
|
||||
id_typ = id_typ,
|
||||
tab_typ = tab_typ,
|
||||
tab_to = table)
|
||||
df = gpd.pd.read_sql_query(sql,con)
|
||||
|
||||
if not df.empty:
|
||||
df.to_sql(
|
||||
name=table, con=con, schema='pr_zh',
|
||||
if_exists='append', index=False
|
||||
)
|
||||
print('INSERT %i correspondances'%df.shape[0])
|
||||
else:
|
||||
print('AUCUNE nouvelles correspondances identifiées')
|
||||
|
||||
|
||||
|
||||
def get_id_t_zh(con,code=None):
|
||||
"""@code : str, list, Serie, Index. Code à 12 characters maximum de la zone humide.
|
||||
"""
|
||||
sql = "SELECT id_zh,code FROM pr_zh.t_zh" # "SELECT id_zh,zh_uuid,code FROM pr_zh.t_zh"
|
||||
if isinstance(code,str):
|
||||
sql += " WHERE code='%s'"%code
|
||||
elif isinstance(code,list) or isinstance(code,gpd.pd.Series) or isinstance(code,gpd.pd.Index):
|
||||
sql += " WHERE code IN {lst_code}".format(lst_code=str(tuple(code)))
|
||||
return gpd.pd.read_sql_query(sql,con)
|
||||
|
||||
|
||||
def to_tzh(df,con):
|
||||
table = 't_zh'
|
||||
|
||||
if isinstance(df,gpd.GeoDataFrame):
|
||||
epsg = df.crs.srs.split(':')[1]
|
||||
df = df.to_wkt()
|
||||
|
||||
df.to_sql(
|
||||
name=table, con=con, schema='pr_zh',
|
||||
if_exists='append', index=False,
|
||||
dtype={
|
||||
'geom': Geometry(srid=epsg)
|
||||
# 'id_lim_list':uuid.SafeUUID
|
||||
}
|
||||
)
|
||||
|
||||
if isinstance(df,gpd.pd.DataFrame):
|
||||
df.geom = gpd.GeoSeries.from_wkt(df.geom)
|
||||
df = df.set_geometry('geom',crs=epsg)
|
||||
|
||||
_cor_zh_area(con,tzh_code=df.code,typ='ZNIEFF1')
|
||||
print('INSERT cor_zh_area ZNIEFF1 OK !')
|
||||
_cor_zh_area(con,tzh_code=df.code,typ='ZNIEFF2')
|
||||
print('INSERT cor_zh_area ZNIEFF2 OK !')
|
||||
_cor_zh_area(con,tzh_code=df.code,typ='DEP')
|
||||
print('INSERT cor_zh_area DEP OK !')
|
||||
_cor_zh_area(con,tzh_code=df.code,typ='COM',cover=True)
|
||||
print('INSERT cor_zh_area COM OK !')
|
||||
_cor_zh_area(con,tzh_code=df.code,typ='ZPS')
|
||||
print('INSERT cor_zh_area ZPS OK !')
|
||||
_cor_zh_area(con,tzh_code=df.code,typ='SIC')
|
||||
print('INSERT cor_zh_area SIC OK !')
|
||||
_cor_zh_area(con,tzh_code=df.code,typ='SRAM')
|
||||
print('INSERT cor_zh_area SRAM OK !')
|
||||
_cor_zh_area(con,tzh_code=df.code,typ='ZICO')
|
||||
print('INSERT cor_zh_area ZICO OK !')
|
||||
|
||||
_cor_zh_(con,tzh_code=df.code,typ='hydro')
|
||||
print('INSERT cor_zh_hydro OK !')
|
||||
_cor_zh_(con,tzh_code=df.code,typ='rb')
|
||||
print('INSERT cor_zh_rb OK !')
|
||||
|
||||
|
||||
|
||||
def to_cor_(con,df,table):
|
||||
|
||||
df.to_sql(
|
||||
name=table,con=con,schema='pr_zh',if_exists='append',index=False
|
||||
)
|
||||
|
||||
|
||||
def to_t_(con,df,table):
|
||||
|
||||
res = gpd.pd.merge(df,get_id_t_zh(con,df.SITE_COD),left_on='SITE_COD',right_on='code')\
|
||||
.drop(columns=['SITE_COD','code'])
|
||||
|
||||
res.to_sql(
|
||||
name=table,con=con,schema='pr_zh',if_exists='append',index=False
|
||||
)
|
||||
|
||||
|
||||
def add_bib_cb(df,con,humidity=None,is_ch=None):
|
||||
"""
|
||||
@df : pd.Dataframe. Tableau à 3 colonnes |lb_code|humidity|is_ch|.
|
||||
@con : engine. Connexion PostgreSQL à la base de données cible.
|
||||
@humidity : str. Caractère d'humidité a attribuer par défault
|
||||
à chaque lb_code du tableau où la colonne est Null.
|
||||
Défaut : None (désactivé)
|
||||
Liste : P (Potentiellement), H (Humide).
|
||||
@is_ch : bool. Caractère de présence/absence de carto d'habitat
|
||||
à attribuer par défault à chaque lb_code du tableau où la colonne est Null.
|
||||
Défaut : None (désactivé)
|
||||
Liste : True (existe), False (n'existe pas).
|
||||
"""
|
||||
table = 'bib_cb'
|
||||
|
||||
if humidity is not None:
|
||||
df.loc[df.humidity.isna(),'humidity'] = humidity.upper()
|
||||
if is_ch is not None:
|
||||
df.loc[df.is_ch.isna(),'is_ch'] = is_ch
|
||||
|
||||
bib_cb = gpd.pd.read_sql_table(table,con,'pr_zh')
|
||||
df = df[~df.lb_code.isin(bib_cb.lb_code)]
|
||||
|
||||
if df.empty:
|
||||
print('NO news data to insert !')
|
||||
else:
|
||||
df.to_sql(
|
||||
name=table,con=con,schema='pr_zh',if_exists='append',index=False
|
||||
)
|
||||
print('INSERT %s news rows !'%df.shape[0])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
from ..MEDWET2Geonat import get_t_zh
|
||||
zh,_cor_lim_list = get_t_zh()
|
||||
Executable
+222
@@ -0,0 +1,222 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
|
||||
from owslib.wfs import WebFeatureService
|
||||
import geopandas as gpd
|
||||
from geoalchemy2 import Geometry
|
||||
from sqlalchemy import dialects
|
||||
|
||||
|
||||
|
||||
def get_wfs(url, layer, bbox=None):
|
||||
from geopandas import read_file
|
||||
from requests import Request
|
||||
|
||||
wfs = WebFeatureService(url=url)
|
||||
item = dict(wfs.items())[layer]
|
||||
crs = item.crsOptions[0].getcode()
|
||||
|
||||
params = dict(service='WFS', version=wfs.version, request='GetFeature',
|
||||
typeName=layer)
|
||||
q = Request('GET', url, params=params).prepare().url
|
||||
|
||||
data = read_file(q, bbox=bbox)
|
||||
data.set_crs(crs=crs, inplace=True)
|
||||
|
||||
if crs != 'EPSG:2154':
|
||||
data.to_crs(epsg=2154, inplace=True)
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def list_layer(url):
|
||||
wfs = WebFeatureService(url=url)
|
||||
lst = list(wfs.contents)
|
||||
return lst
|
||||
|
||||
|
||||
def Polygons_to_MultiPolygon(df,geom_col=None):
|
||||
from shapely.geometry import MultiPolygon
|
||||
from pandas import concat
|
||||
|
||||
if not geom_col:
|
||||
geom_col = df.geometry.name
|
||||
|
||||
df = df.copy()
|
||||
multi = df.loc[df.geom_type=='MultiPolygon'].copy()
|
||||
poly = df.loc[df.geom_type=='Polygon'].copy()
|
||||
poly[geom_col] = [MultiPolygon([geom]) for geom in df.loc[df.geom_type=='Polygon',geom_col] ]
|
||||
df = concat([multi,poly])
|
||||
df.sort_index(inplace=True)
|
||||
return df
|
||||
|
||||
|
||||
def ref_geo(type_code,con):
|
||||
sql = """
|
||||
SELECT l.* FROM ref_geo.l_areas l
|
||||
JOIN ref_geo.bib_areas_types bib USING (id_type)
|
||||
WHERE bib.type_code='{typ}'
|
||||
""".format(typ=type_code)
|
||||
return gpd.read_postgis(sql,con)
|
||||
|
||||
|
||||
def to_lareas(df,dic,layer,con,dtypes={}):
|
||||
|
||||
id_type = gpd.pd.read_sql_query("""
|
||||
SELECT id_type FROM ref_geo.bib_areas_types WHERE type_code='%s'
|
||||
"""%layer,con).id_type.values[0]
|
||||
|
||||
ref = ref_geo(layer,con)
|
||||
|
||||
df.rename(columns=dic, inplace=True)
|
||||
df = Polygons_to_MultiPolygon(
|
||||
df[~df.area_name.isin(ref.area_name)]
|
||||
)
|
||||
df.rename_geometry('geom', inplace=True)
|
||||
del_col = df.columns[~df.columns.isin(['geom',*[*set(dic.values())]])]
|
||||
df.drop(columns=del_col, inplace=True)
|
||||
|
||||
df['id_type'] = id_type
|
||||
df['geojson_4326'] = df.to_crs(4326).geom.__geo_interface__['features']
|
||||
df['geojson_4326'] = [x['geometry'] for x in df['geojson_4326']]
|
||||
df['centroid'] = 'SRID=2154;'+df.geom.centroid.to_wkt()
|
||||
df['enable'] = True
|
||||
|
||||
if df.empty:
|
||||
print('AUCUN nouveaux zonages identifiés')
|
||||
else:
|
||||
df.to_postgis(
|
||||
name='l_areas',
|
||||
con=con,
|
||||
schema='ref_geo',
|
||||
if_exists='append',
|
||||
index=False,
|
||||
index_label=None,
|
||||
chunksize=None,
|
||||
dtype={
|
||||
'centroid': Geometry(geometry_type='POINT',srid=2154),
|
||||
**dtypes
|
||||
},
|
||||
)
|
||||
print('INSERT %i zones'%df.shape[0])
|
||||
|
||||
|
||||
|
||||
def update_larea(con,layer,df,cols_updt=[],on='area_name'):
|
||||
from pycen import update_to_sql
|
||||
table = 'l_areas'
|
||||
# idtyp = tuple(df.id_type.unique())
|
||||
|
||||
idtyp = gpd.pd.read_sql_query("""
|
||||
SELECT id_type FROM ref_geo.bib_areas_types WHERE type_code='%s'
|
||||
"""%layer,con).id_type.values[0]
|
||||
|
||||
# if len(idtyp) > 1:
|
||||
# where = 'in %s'%tuple(df.id_type.unique())
|
||||
# else :
|
||||
where = '=%i'%idtyp
|
||||
|
||||
sql = 'SELECT id_area, %s FROM ref_geo.l_areas WHERE id_type %s;'%(on,where)
|
||||
larea = gpd.pd.read_sql_query(sql,con)
|
||||
|
||||
to_updt = df.merge(larea, on=on)
|
||||
update_to_sql(
|
||||
df=to_updt[['id_area',*cols_updt]],
|
||||
con=con,
|
||||
table_name=table,
|
||||
schema_name='ref_geo',
|
||||
key_name='id_area',
|
||||
)
|
||||
|
||||
|
||||
carmen = 'https://ws.carmencarto.fr/WFS/119/fxx_inpn?'
|
||||
dic_layer = {
|
||||
# ref_geo.bib_areas_types.type_code : layer_name from list_layer
|
||||
'ZNIEFF2':'Znieff2',
|
||||
'ZNIEFF1':'Znieff1',
|
||||
'APB':'Arretes_de_protection_de_biotope',
|
||||
'RNN':'Reserves_naturelles_nationales',
|
||||
'RNR':'Reserves_naturelles_regionales',
|
||||
'ZPS':'Zones_de_protection_speciale',
|
||||
'SIC':'Sites_d_importance_communautaire',
|
||||
'ZICO':'ZICO',
|
||||
'RNCFS':'Reserves_nationales_de_chasse_et_faune_sauvage',
|
||||
'RIPN':'Reserves_Integrales_de_Parcs_Nationaux', # NO zone in 74
|
||||
'SCEN':'Terrains_acquis_des_Conservatoires_des_espaces_naturels', # Pas fait ;
|
||||
# 'SCL':'',
|
||||
# 'PNM':'',
|
||||
'PNR':'Parcs_naturels_regionaux',
|
||||
'RBIOL':'Reserves_biologiques',
|
||||
'RBIOS':'Reserves_de_la_biosphere', # NO zone in 74
|
||||
# 'RNC':'',
|
||||
'SRAM':'Sites_Ramsar',
|
||||
# 'AA':'',
|
||||
# 'ZSC':'',
|
||||
# 'PSIC':'',
|
||||
# 'PEC':'',
|
||||
# ...
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
from sqlalchemy import create_engine # pour lecture de la bd Géonature
|
||||
from sqlalchemy.engine import URL
|
||||
usr = 'XXXXX'
|
||||
pdw = 'XXXXX'
|
||||
host= 'XXXXX'
|
||||
bd = 'XXXXX'
|
||||
|
||||
eng = URL.create('postgresql+psycopg2',username=usr,password=pdw,host=host,database=bd)
|
||||
conn = create_engine(eng)
|
||||
# from pycen import con_gn as conn
|
||||
|
||||
ref = ref_geo('DEP',conn)
|
||||
ter = ref[ref.area_code=='74']
|
||||
|
||||
layer = 'Znieff1'
|
||||
# layer_name = dic_layer[layer]
|
||||
zon = get_wfs(carmen,layer,bbox=ter.unary_union)
|
||||
zon = zon[zon.intersects(ter.unary_union)]
|
||||
|
||||
dic_cols = {
|
||||
'ID_MNHN':'area_code',
|
||||
'NOM':'area_name',
|
||||
'URL':'source',
|
||||
}
|
||||
|
||||
dtyp = {}
|
||||
if layer in ['ZPS','SIC']:
|
||||
dic_cols = {**dic_cols,**{'SITECODE':'area_code','SITENAME':'area_name',}}
|
||||
elif layer == 'RNN':
|
||||
dic_cols = {**dic_cols,**{'URL_FICHE':'source','NOM_SITE':'area_name',}}
|
||||
elif layer == 'ZICO':
|
||||
dic_cols = {**dic_cols,**{'ID_SPN':'area_code'}}
|
||||
elif layer == 'SCEN':
|
||||
zon.NOM = zon.NOM.str.title()
|
||||
elif layer == 'RBIOL':
|
||||
dic_cols = {
|
||||
**dic_cols,
|
||||
**{'URL_FICHE':'source','NOM_SITE':'area_name',
|
||||
'comment':'comment','additional_data':'additional_data',}
|
||||
}
|
||||
zon['comment'] = 'Réserves biologiques ' + zon.CODE_R_ENP.replace(
|
||||
{'I': 'intégrale','D' : 'dirigée'}
|
||||
)
|
||||
zon['additional_data'] = [
|
||||
str({ zon.CODE_R_ENP.name : x }).replace("\'",'\"')
|
||||
for x in zon.CODE_R_ENP
|
||||
]
|
||||
dtyp = {'additional_data':dialects.postgresql.JSONB}
|
||||
|
||||
|
||||
# IF UPDATE DATA
|
||||
# update_larea(conn,layer,zon.rename(columns=dic_cols),['comment'],on='area_code')
|
||||
|
||||
|
||||
to_lareas(
|
||||
df=zon,
|
||||
dic=dic_cols,
|
||||
layer=layer,
|
||||
con=conn,
|
||||
dtypes=dtyp
|
||||
)
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
import geopandas as gpd
|
||||
from pycen import update_to_sql, con_gn, con as con_aza
|
||||
|
||||
def get_sdr_eau(con=con_aza):
|
||||
return gpd.read_postgis('SELECT * FROM ref_territoire.sdr_ressources_en_eau', con)
|
||||
|
||||
def get_zh_to_update(con=con_gn):
|
||||
return gpd.read_postgis("SELECT id_zh,code,ST_TRANSFORM(geom,2154) geom FROM pr_zh.t_zh WHERE code ilike '38CENIS%%'", con)
|
||||
|
||||
def change_code(zh,sdr,con=con_gn):
|
||||
res = zh.sjoin(sdr[['code_tr','geom']]).drop(columns='index_right')
|
||||
code_max = gpd.pd.read_sql(
|
||||
"""SELECT MAX(SUBSTRING(code,5,4)::int) val_max, SUBSTRING(code,3,2) code_tr
|
||||
FROM pr_zh.t_zh WHERE code ilike ANY(ARRAY['%%{}%%'])
|
||||
GROUP BY SUBSTRING(code,3,2)""".format(
|
||||
"%%','%%".join(res.code_tr.unique().tolist())
|
||||
),con_gn
|
||||
)
|
||||
|
||||
for i,r in code_max.iterrows():
|
||||
res.loc[res.code_tr==r.code_tr,'code_new'] = ['38'+row.code_tr+str(r.val_max+i+1).zfill(4) for i,row in res.iterrows()]
|
||||
|
||||
update_to_sql(
|
||||
res[['id_zh','code_new']].rename(columns={'code_new':'code'}),
|
||||
con_gn,
|
||||
't_zh',
|
||||
'pr_zh',
|
||||
'id_zh'
|
||||
)
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
sdr = get_sdr_eau(con=con_aza)
|
||||
zh = get_zh_to_update(con=con_gn)
|
||||
change_code(zh,sdr,con=con_gn)
|
||||
Executable
+31
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
|
||||
import os
|
||||
import pandas as pd
|
||||
import geopandas as gpd
|
||||
from geoalchemy2 import Geometry
|
||||
|
||||
def move2filedir():
|
||||
'''
|
||||
Orientation vers le répertoire où est localiser le script zh2gn.py
|
||||
'''
|
||||
dir_path = os.path.dirname(os.path.realpath(__file__))
|
||||
os.chdir(dir_path)
|
||||
|
||||
# print(os.getcwd())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# move2filedir()
|
||||
os.chdir('/home/colas/Documents/13_CEN74/medwet2gn_ZH/tools')
|
||||
|
||||
PATH = '../template/'
|
||||
FILE_DATA = 'BDD_ZH_CCPR.xlsx'
|
||||
GEO_DATA = 'ZH_CCPR_finalisées/ZH_asters_finalisées.shp'
|
||||
|
||||
data = pd.read_excel(PATH+FILE_DATA,'formulaire_type')
|
||||
geo = gpd.read_file(PATH+GEO_DATA)
|
||||
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import geopandas as gpd
|
||||
from pycen import con_gn, update_to_sql
|
||||
from os import path
|
||||
|
||||
PATH = '/media/cgeier/SRV/FICHIERS/OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/ETUDES/Zones humides/2026/HUEZ_Modification zonage_ajouts ZH/'
|
||||
file = 'HUEZ_modifs 2026_AG.geojson'
|
||||
auth_id = 45
|
||||
code_zh = 'site_code'
|
||||
|
||||
zh = gpd.read_file(path.join(PATH,file))[[code_zh,'geometry']]
|
||||
|
||||
if code_zh != 'code':
|
||||
zh.rename(columns={code_zh:'code'}, inplace=True)
|
||||
|
||||
if zh.crs.srs.lower() != 'epsg:4326':
|
||||
zh.to_crs(4326,inplace=True)
|
||||
|
||||
if zh.geometry.name != 'geom':
|
||||
zh.rename_geometry('geom', inplace=True)
|
||||
|
||||
zh['area'] = zh.to_crs(2154).area / 10000
|
||||
zh['update_author'] = auth_id
|
||||
|
||||
if zh.code.is_unique:
|
||||
update_to_sql(zh,con_gn,'t_zh','pr_zh','code')
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import geopandas as gpd
|
||||
import os
|
||||
|
||||
PATH = '/media/cgeier/SRV/FICHIERS/SITES/SITES GERES/CHAR_CHARVAS/Scientifique-technique/Faune/AMPHIBIENS'
|
||||
file = 'Inventaires amphibiens Charvas 2011.xlsx'
|
||||
|
||||
|
||||
df = gpd.pd.read_excel(os.path.join(PATH,file))
|
||||
df.DATE = df.DATE.replace('\n','',regex=True)
|
||||
df["NOM DU POINT D'EAU"] = df["NUMER O DU POINT D'EAU 2"].astype(str)+' - '+df["NOM DU POINT D'EAU"]
|
||||
df[['LOCALISATION PRECISE (X)','LOCALISATION PRECISE (Y)']] = df[['LOCALISATION PRECISE (X)','LOCALISATION PRECISE (Y)']].replace({',':'.',' ':'','\n':''},regex=True)
|
||||
df['X_93'] = gpd.points_from_xy(df['LOCALISATION PRECISE (X)'],df['LOCALISATION PRECISE (Y)'],crs=27572).to_crs(2154).x
|
||||
df['Y_93'] = gpd.points_from_xy(df['LOCALISATION PRECISE (X)'],df['LOCALISATION PRECISE (Y)'],crs=27572).to_crs(2154).y
|
||||
|
||||
df.to_excel(os.path.join(PATH,file),index=False)
|
||||
|
||||
|
||||
df = gpd.pd.read_excel(os.path.join(PATH,file))
|
||||
@@ -0,0 +1,22 @@
|
||||
import pandas as pd
|
||||
from os import path
|
||||
from pycen import con_gn
|
||||
|
||||
# PARAMS
|
||||
PATH = '/home/cgeier/Téléchargements'
|
||||
file = 'donnees_ebms_platiere2025.csv'
|
||||
|
||||
# READ DATA
|
||||
tax = pd.read_sql('SELECT * FROM taxonomie.taxref where cd_nom=cd_ref',con_gn)
|
||||
df = pd.read_csv(path.join(PATH,file),index_col=0)
|
||||
|
||||
# MAKE DICTIONARY
|
||||
dic_tax_lbno = dict(zip(tax.lb_nom,tax.cd_ref.astype(str)))
|
||||
|
||||
# MANIP DATA
|
||||
df['cd_ref'] = df['Preferred Species Name'].copy()
|
||||
df.replace({'cd_ref':dic_tax_lbno},inplace=True)
|
||||
|
||||
df['cd_ref'] = df['cd_ref'].astype(int)
|
||||
|
||||
df.to_csv(path.join(PATH,file))
|
||||
@@ -0,0 +1,60 @@
|
||||
import pandas as pd
|
||||
from os import path
|
||||
from pycen import con_gn
|
||||
|
||||
# PARAMS
|
||||
PATH = '/media/cgeier/SRV/FICHIERS/OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/SITES/Lone de la Negria/Faune-Flore/Données partenaires/1.Données nettoyées/Golf Club Lyon Faune Flore Oxalis 2021/'
|
||||
file = 'Golf Club Lyon Faune Flore Oxalis 2021.csv'
|
||||
|
||||
# READ DATA
|
||||
tax = pd.read_sql('SELECT * FROM taxonomie.taxref',con_gn)
|
||||
df = pd.read_csv(path.join(PATH,file),index_col=0)
|
||||
|
||||
# MAKE DICTIONARY
|
||||
dic_tax_comp = dict(zip(tax.nom_complet,tax.cd_ref.astype(str)))
|
||||
dic_tax_vali = dict(zip(tax.nom_valide,tax.cd_ref.astype(str)))
|
||||
dic_tax_lbno = dict(zip(tax.lb_nom,tax.cd_ref.astype(str)))
|
||||
dic_tax_vern = dict(zip(tax.nom_vern,tax.cd_ref.astype(str)))
|
||||
|
||||
# MANIP DATA
|
||||
df['cd_ref'] = df.NOMCITE.copy()
|
||||
df.replace({'cd_ref':dic_tax_comp},inplace=True)
|
||||
df.replace({'cd_ref':dic_tax_vali},inplace=True)
|
||||
df.replace({'cd_ref':dic_tax_lbno},inplace=True)
|
||||
df.replace({'cd_ref':dic_tax_vern},inplace=True)
|
||||
|
||||
df['tmp'] = df.COMMENT.str.lower().copy()
|
||||
df.tmp.replace({
|
||||
'â':'a',
|
||||
'ê':'e'
|
||||
}, regex=True)
|
||||
|
||||
df.loc[df.tmp.str.contains("mâle",na=False),'sexe'] = "Mâle"
|
||||
df.loc[df.tmp.str.contains("adulte",na=False),'stade_vie'] = "Adulte"
|
||||
df.loc[df.tmp.str.contains("imago",na=False),'stade_vie'] = "Imago"
|
||||
df.loc[df.tmp.str.contains("larve",na=False),'stade_vie'] = "Larve"
|
||||
df.loc[df.tmp.str.contains("tetard",na=False),'stade_vie'] = "Têtard"
|
||||
df.loc[df.tmp.str.contains("immature",na=False),'stade_vie'] = "Immature"
|
||||
df.loc[df.tmp.str.contains("flor",na=False),'stade_vie'] = "Fleur"
|
||||
df.loc[df.tmp.str.contains("cris",na=False),'comportement'] = "Cris"
|
||||
df.loc[df.tmp.str.contains("cht",na=False),'comportement'] = "Chant"
|
||||
df.loc[df.tmp.str.contains("chant",na=False),'comportement'] = "Chant"
|
||||
df.loc[df.tmp.str.contains("transit",na=False),'comportement'] = "Passage en vol"
|
||||
df.loc[df.tmp.str.contains("vol",na=False),'comportement'] = "Passage en vol"
|
||||
df.loc[df.tmp.str.contains("chasse",na=False),'comportement'] = "Chasse/alimentation"
|
||||
df.loc[df.tmp.str.contains("andem",na=False),'comportement'] = "Tandem"
|
||||
df.loc[df.tmp.str.contains("ponte",na=False),'obj_denombrement'] = "Ponte"
|
||||
|
||||
df.sexe.fillna('Non renseigné', inplace=True)
|
||||
df.stade_vie.fillna('Inconnu', inplace=True)
|
||||
df.comportement.fillna('Inconnu', inplace=True)
|
||||
df.obj_denombrement.fillna('Individu', inplace=True)
|
||||
|
||||
df['typ_denombre'] = 'Compté'
|
||||
df['etat_bio'] = 'Observé vivant'
|
||||
df['tech_obs'] = 'Vu'
|
||||
df['OBSE_NOMBR'] = df.OBSE_NOMBR.astype(int)
|
||||
df.loc[~df.PRECISGEO.isna(),'precisgeo'] = df[~df.PRECISGEO.isna()].PRECISGEO.astype(int).astype(str)
|
||||
|
||||
del df['tmp']
|
||||
df.to_csv(path.join(PATH,'Golf Club Lyon Faune Flore Oxalis 2021 (remanié).csv'))
|
||||
@@ -0,0 +1,73 @@
|
||||
|
||||
|
||||
# UsersHub
|
||||
VERSION=2.4.7
|
||||
cd
|
||||
wget https://github.com/PnX-SI/UsersHub/archive/$VERSION.zip
|
||||
unzip $VERSION.zip
|
||||
rm $VERSION.zip
|
||||
sudo rm -R ~/usershub_old
|
||||
mv /home/`whoami`/usershub/ /home/`whoami`/usershub_old/
|
||||
mv UsersHub-$VERSION /home/`whoami`/usershub/
|
||||
cp /home/`whoami`/usershub_old/config/config.py /home/`whoami`/usershub/config/config.py
|
||||
cp /home/`whoami`/usershub_old/config/settings.ini /home/`whoami`/usershub/config/settings.ini
|
||||
cd usershub
|
||||
./install_app.sh
|
||||
|
||||
# Monitoring
|
||||
VERSION=1.0.3
|
||||
cd
|
||||
wget https://github.com/PnX-SI/gn_module_monitoring/archive/$VERSION.zip
|
||||
unzip $VERSION.zip
|
||||
rm $VERSION.zip
|
||||
sudo rm -R ~/gn_module_monitoring_old
|
||||
mv ~/gn_module_monitoring/ ~/gn_module_monitoring_old/
|
||||
mv ~/gn_module_monitoring-$VERSION ~/gn_module_monitoring
|
||||
|
||||
# Export
|
||||
VERSION=1.7.2
|
||||
cd
|
||||
wget https://github.com/PnX-SI/gn_module_export/archive/$VERSION.zip
|
||||
unzip $VERSION.zip
|
||||
rm $VERSION.zip
|
||||
sudo rm -R ~/gn_module_export_old
|
||||
mv ~/gn_module_export/ ~/gn_module_export_old/
|
||||
mv ~/gn_module_export-$VERSION ~/gn_module_export
|
||||
|
||||
# ~/gn_module_export_old/backend/templates/swagger
|
||||
# ZH
|
||||
VERSION=1.4.0
|
||||
cd
|
||||
wget https://github.com/PnX-SI/gn_module_ZH/archive/$VERSION.zip
|
||||
unzip $VERSION.zip
|
||||
rm $VERSION.zip
|
||||
sudo rm -R ~/gn_module_ZH_old
|
||||
mv ~/gn_module_ZH/ ~/gn_module_ZH_old/
|
||||
mv ~/gn_module_ZH-$VERSION ~/gn_module_ZH
|
||||
|
||||
# GeoNature
|
||||
VERSION=2.15.4
|
||||
wget https://github.com/PnX-SI/GeoNature/archive/$VERSION.zip
|
||||
unzip $VERSION.zip
|
||||
rm $VERSION.zip
|
||||
sudo rm -R ~/geonature_old
|
||||
mv ~/geonature/ ~/geonature_old/
|
||||
mv ~/GeoNature-$VERSION ~/geonature/
|
||||
cd ~/geonature
|
||||
./install/migration/migration.sh 2>&1 | tee install/migration/migration.log
|
||||
|
||||
1. Remove API_TAXHUB
|
||||
2. Modifier taxhub_url IN /geonature/backend/media/mobile/occtax/settings.json
|
||||
3. geonature db downgrade taxhub-standalone@base
|
||||
4. Check media taxhub IN GeoNature THEN rm ~/taxhub
|
||||
5. Modifier config Apache TaxHub
|
||||
``` sh
|
||||
sudo nano /etc/apache2/conf-available/taxhub.conf
|
||||
|
||||
# Cas où TaxHub et GeoNature sont sur le même sous-domaine
|
||||
RewriteEngine on
|
||||
RewriteRule "^/taxhub/static/medias/(.+)" "/geonature/api/medias/taxhub/$1" [R,L]
|
||||
```
|
||||
~6. geonature monitorings update_module_available_permissions
|
||||
~7. RUN data/upgrade_modules_permissions.sql
|
||||
8. Update VIEWS Monitorings
|
||||
@@ -0,0 +1,128 @@
|
||||
|
||||
# Monitoring
|
||||
VERSION=1.1.0
|
||||
cd
|
||||
wget https://github.com/PnX-SI/gn_module_monitoring/archive/$VERSION.zip
|
||||
unzip $VERSION.zip
|
||||
rm $VERSION.zip
|
||||
sudo rm -R ~/gn_module_monitoring_old
|
||||
mv ~/gn_module_monitoring/ ~/gn_module_monitoring_old/
|
||||
mv ~/gn_module_monitoring-$VERSION ~/gn_module_monitoring
|
||||
|
||||
# Export
|
||||
VERSION=1.8.0
|
||||
cd
|
||||
wget https://github.com/PnX-SI/gn_module_export/archive/$VERSION.zip
|
||||
unzip $VERSION.zip
|
||||
rm $VERSION.zip
|
||||
sudo rm -R ~/gn_module_export_old
|
||||
mv ~/gn_module_export/ ~/gn_module_export_old/
|
||||
mv ~/gn_module_export-$VERSION ~/gn_module_export
|
||||
|
||||
# ~/gn_module_export_old/backend/templates/swagger
|
||||
# DASHBOARD
|
||||
VERSION=1.6.0
|
||||
cd
|
||||
wget https://github.com/PnX-SI/gn_module_dashboard/archive/$VERSION.zip
|
||||
unzip $VERSION.zip
|
||||
rm $VERSION.zip
|
||||
sudo rm -R ~/gn_module_dashboard_old
|
||||
mv ~/gn_module_dashboard/ ~/gn_module_dashboard_old/
|
||||
mv ~/gn_module_dashboard-$VERSION ~/gn_module_dashboard
|
||||
|
||||
# GeoNature
|
||||
VERSION=2.16.2
|
||||
wget https://github.com/PnX-SI/GeoNature/archive/$VERSION.zip
|
||||
unzip $VERSION.zip
|
||||
rm $VERSION.zip
|
||||
sudo rm -R ~/geonature_old
|
||||
mv ~/geonature/ ~/geonature_old/
|
||||
mv ~/GeoNature-$VERSION ~/geonature/
|
||||
cd ~/geonature
|
||||
./install/migration/migration.sh 2>&1 | tee install/migration/migration.log
|
||||
|
||||
|
||||
|
||||
# LOG WARNING
|
||||
Considering dependency proxy for proxy_http:
|
||||
Module proxy already enabled
|
||||
Module proxy_http already enabled
|
||||
Considering dependency filter for deflate:
|
||||
Module filter already enabled
|
||||
Module deflate already enabled
|
||||
AH00112: Warning: DocumentRoot [/home/geonatureadmin/geonature/frontend/dist] does not exist
|
||||
Syntax OK
|
||||
Set API_ENDPOINT to https://geonature.cen-isere.fr/geonature/api in frontend configuration file...
|
||||
Mise à jour des fichiers de configuration frontend et rebuild du frontend…
|
||||
Génération de la configuration du frontend :
|
||||
|
||||
# LOG ERROR
|
||||
Mise à jour de la base de données…
|
||||
/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/alembic/config.py:560: DeprecationWarning: No path_separator found in configuration; falling back to legacy splitting on spaces/commas for version_locations. Consider adding path_separator=os to Alembic config.
|
||||
util.warn_deprecated(
|
||||
/home/geonatureadmin/geonature/backend/geonature/migrations/versions/7dfd0a813f86_insert_inpn_sensitivity_referential.py:14: DeprecationWarning: The distutils package is deprecated and slated for removal in Python 3.12. Use setuptools or check PEP 632 for potential alternatives
|
||||
from distutils.util import strtobool
|
||||
/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/alembic/config.py:560: DeprecationWarning: No path_separator found in configuration; falling back to legacy splitting on spaces/commas for version_locations. Consider adding path_separator=os to Alembic config.
|
||||
util.warn_deprecated(
|
||||
INFO [alembic.runtime.migration] Context impl PostgresqlImpl.
|
||||
INFO [alembic.runtime.migration] Will assume transactional DDL.
|
||||
/home/geonatureadmin/geonature/backend/geonature/migrations/versions/7dfd0a813f86_insert_inpn_sensitivity_referential.py:14: DeprecationWarning: The distutils package is deprecated and slated for removal in Python 3.12. Use setuptools or check PEP 632 for potential alternatives
|
||||
from distutils.util import strtobool
|
||||
ERROR [flask_migrate] Error: Can't locate revision identified by 'd85b87a1ca62'
|
||||
/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/utils_flask_sqla/commands.py:89: DeprecationWarning: The 'db' attribute is deprecated and will be removed in Flask-SQLAlchemy 3.1. The extension is registered directly as 'app.extensions["sqlalchemy"]'.
|
||||
db = current_app.extensions["sqlalchemy"].db
|
||||
/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/alembic/config.py:560: DeprecationWarning: No path_separator found in configuration; falling back to legacy splitting on spaces/commas for version_locations. Consider adding path_separator=os to Alembic config.
|
||||
util.warn_deprecated(
|
||||
/home/geonatureadmin/geonature/backend/geonature/migrations/versions/7dfd0a813f86_insert_inpn_sensitivity_referential.py:14: DeprecationWarning: The distutils package is deprecated and slated for removal in Python 3.12. Use setuptools or check PEP 632 for potential alternatives
|
||||
from distutils.util import strtobool
|
||||
Traceback (most recent call last):
|
||||
File "/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/alembic/script/base.py", line 217, in _catch_revision_errors
|
||||
yield
|
||||
File "/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/alembic/script/base.py", line 285, in get_all_current
|
||||
return cast(Set[Script], self.revision_map._get_all_current(id_))
|
||||
File "/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/alembic/script/revision.py", line 1531, in _get_all_current
|
||||
top_revs = set(self.get_revisions(id_))
|
||||
File "/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/alembic/script/revision.py", line 542, in get_revisions
|
||||
return sum([self.get_revisions(id_elem) for id_elem in id_], ())
|
||||
File "/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/alembic/script/revision.py", line 542, in <listcomp>
|
||||
return sum([self.get_revisions(id_elem) for id_elem in id_], ())
|
||||
File "/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/alembic/script/revision.py", line 565, in get_revisions
|
||||
return tuple(
|
||||
File "/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/alembic/script/revision.py", line 566, in <genexpr>
|
||||
self._revision_for_ident(rev_id, branch_label)
|
||||
File "/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/alembic/script/revision.py", line 637, in _revision_for_ident
|
||||
raise ResolutionError(
|
||||
alembic.script.revision.ResolutionError: No such revision or branch 'd85b87a1ca62'
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "/home/geonatureadmin/geonature/backend/venv/bin/geonature", line 7, in <module>
|
||||
sys.exit(main())
|
||||
File "/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/click/core.py", line 1161, in __call__
|
||||
return self.main(*args, **kwargs)
|
||||
File "/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/click/core.py", line 1082, in main
|
||||
rv = self.invoke(ctx)
|
||||
File "/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/click/core.py", line 1697, in invoke
|
||||
return _process_result(sub_ctx.command.invoke(sub_ctx))
|
||||
File "/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/click/core.py", line 1697, in invoke
|
||||
return _process_result(sub_ctx.command.invoke(sub_ctx))
|
||||
File "/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/click/core.py", line 1443, in invoke
|
||||
return ctx.invoke(self.callback, **ctx.params)
|
||||
File "/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/click/core.py", line 788, in invoke
|
||||
return __callback(*args, **kwargs)
|
||||
File "/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/click/decorators.py", line 33, in new_func
|
||||
return f(get_current_context(), *args, **kwargs)
|
||||
File "/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/flask/cli.py", line 400, in decorator
|
||||
return ctx.invoke(f, *args, **kwargs)
|
||||
File "/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/click/core.py", line 788, in invoke
|
||||
return __callback(*args, **kwargs)
|
||||
File "/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/utils_flask_sqla/commands.py", line 97, in autoupgrade
|
||||
current_heads = set(map(lambda rev: rev.revision, script.get_all_current(current_heads)))
|
||||
File "/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/alembic/script/base.py", line 284, in get_all_current
|
||||
with self._catch_revision_errors():
|
||||
File "/usr/lib/python3.10/contextlib.py", line 153, in __exit__
|
||||
self.gen.throw(typ, value, traceback)
|
||||
File "/home/geonatureadmin/geonature/backend/venv/lib/python3.10/site-packages/alembic/script/base.py", line 249, in _catch_revision_errors
|
||||
raise util.CommandError(resolution) from re
|
||||
alembic.util.exc.CommandError: Can't locate revision identified by 'd85b87a1ca62'
|
||||
@@ -0,0 +1,68 @@
|
||||
from pycen import con_gn
|
||||
import pycen
|
||||
import pandas as pd
|
||||
import os
|
||||
|
||||
def get_taxref(con):
|
||||
sql = '''SELECT cd_nom,cd_ref,nom_complet,lb_nom FROM taxonomie.taxref;'''
|
||||
return pd.read_sql_query(sql,con)
|
||||
|
||||
def get_cd_nom(con,lb):
|
||||
sql = '''SELECT cd_nom,cd_ref,nom_complet,lb_nom FROM taxonomie.taxref
|
||||
WHERE nom_complet='{label}' or nom_complet ilike '%%{label}%%';'''.format(label=lb)
|
||||
return pd.read_sql_query(sql,con)
|
||||
|
||||
|
||||
dict_taxo = {
|
||||
'Brachythecium rutabulum (Hedw.) Schimp. var. rutabulum':434653,
|
||||
'Hypnum cupressiforme Hedw. var. cupressiforme':434662,
|
||||
'Orthotrichum affine Brid.':5014,
|
||||
'Plagiomnium cuspidatum ( Hedw.) T.J.Kop.':4934,
|
||||
'Leucodon sciuroides (Hedw.) Schwägr var. sciuroides':None,
|
||||
'Brachythecium albicans (Hedw. ) Schimp.':5807,
|
||||
'Phascum cuspidatum Hedw. var. cuspidatum':434590,
|
||||
'Syntrichia ruralis (Hedw.) F.Weber & D.Mohr var. ruralis':434602,
|
||||
'Brachythecium salebrosum (Hoffm. ex F.Weber & D.Mohr) Schimp. nom. cons.':5828,
|
||||
'Lunularia cruciata (L.) Lindb.':6167,
|
||||
'Rhynchostegium megapolitanum (Blandow ex F.Weber & D.Mohr) Schimp.':5908,
|
||||
'Orthotrichum cupulatum Hoffm. ex Brid. var. riparium Huebener':434606,
|
||||
'Syntrichia montana Nees var. montana':434600,
|
||||
'Abietinella abietina (Hedw.) M.Fleisch. var. abietina':434642,
|
||||
'Ceratodon purpureus (Hedw.) Brid. subsp. purpureus':434562,
|
||||
'Hedwigia ciliata (Hedw.) P.Beauv. var. leucophaea Bruch & Schimp.':434612,
|
||||
'Hypnum cupressiforme Hedw. var. filiforme Brid.':6046,
|
||||
'Hedwigia ciliata (Hedw.) P.Beauv. var. ciliata':434611,
|
||||
'Orthotrichum speciosum Nees var. speciosum':None,
|
||||
'Brachytheciastrum velutinum (Hedw.) Ignatov & Huttunen var. velutinum':434659
|
||||
}
|
||||
dict_taxo2 = {
|
||||
'Leucodon sciuroides (Hedw.) Schwägr var. sciuroides':5104,
|
||||
'Orthotrichum speciosum Nees var. speciosum':5041,
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
PATH = '/media/cgeier/Disk2/tmp'
|
||||
file = '(RNN Platière) Inventaire Bryophytes L93.csv'
|
||||
file_out = '(RNN Platière) Inventaire Bryophytes L93 (correct cdnom).csv'
|
||||
df = pd.read_csv(os.path.join(PATH,file),encoding='Windows-1252',sep=';')
|
||||
tax = get_taxref(con_gn)
|
||||
|
||||
for n in df.NOM_COMPLET.unique():
|
||||
|
||||
res = get_cd_nom(con_gn,n)
|
||||
df.loc[df.NOM_COMPLET==n,'CD_NOM2'] = res.cd_nom.values[0] if not res.empty else None
|
||||
if res.shape[0]>1:
|
||||
print(n)
|
||||
|
||||
df.CD_NOM2 = df.CD_NOM2.astype('Int64')
|
||||
cd_isna = df.CD_NOM2.isna()
|
||||
df.loc[cd_isna,'CD_NOM2'] = df.loc[cd_isna,'NOM_COMPLET'].replace(dict_taxo).astype('Int64')
|
||||
cd_isna = df.CD_NOM2.isna()
|
||||
df.loc[cd_isna,'remarque_obs'] = "Nom complet d'origine : "+df[cd_isna].NOM_COMPLET
|
||||
df.loc[cd_isna,'CD_NOM2'] = df.loc[cd_isna,'NOM_COMPLET'].replace(dict_taxo2).astype('Int64')
|
||||
df.CD_NOM2 = df.CD_NOM2.astype(int)
|
||||
df.to_csv(os.path.join(PATH,file_out),encoding='Windows-1252',sep=';',index=False)
|
||||
|
||||
|
||||
@@ -0,0 +1,428 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
|
||||
import geopandas as gpd
|
||||
from shapely import wkt
|
||||
from shapely.ops import nearest_points,Polygon,Point,LineString
|
||||
from os import path,listdir
|
||||
|
||||
PATH = '/home/colas/Documents/9_PROJETS/6_GEONATURE/MIGRATION/PLATIERE/4_JDD_Faune_Mailles_A_Importer'
|
||||
FILE_DATA = '(CNPE) Chrono-inventaires entomologiques Orthopteres.csv'
|
||||
FILE_DATA = '(CNPE) Observations Opportunistes Orthopteres.csv'
|
||||
FILE_EF_0 = 'Ensembles_Fonctionnels_Platière_Lambert93.shp'
|
||||
FILE_GRID = 'grillegographiquecomplte/Grille_Geographique_Lignes_Complete_WGS84.shp'
|
||||
dictef_cols = {
|
||||
'code_ensemble':'code_ensem',
|
||||
**dict.fromkeys(['ensemble_fonctionnel','nom_ensemble'],'nom_ensemb'),
|
||||
}
|
||||
ef0 = gpd.read_file(path.join(PATH,'..',FILE_EF_0),encoding='UTF-8')
|
||||
ef1 = gpd.read_file(path.join(PATH,'..','ensemblefonctionnelscomplte','ensemble fonc baix.TAB'),encoding='UTF-8')
|
||||
ef2 = gpd.read_file(path.join(PATH,'..','ensemblefonctionnelscomplte','ensemble fonc donzère.TAB'),encoding='UTF-8')
|
||||
ef3 = gpd.read_file(path.join(PATH,'..','ensemblefonctionnelscomplte','Ensembles fonctionnels montélimar.TAB'),encoding='UTF-8')
|
||||
ef4 = gpd.read_file(path.join(PATH,'..','ensemblefonctionnelscomplte','Secteurs obs oiseaux montélimar.TAB'),encoding='UTF-8')
|
||||
for e in [ef1,ef2,ef3,ef4]:
|
||||
e.columns = e.columns.str.lower()
|
||||
e.rename(columns=dictef_cols,inplace=True)
|
||||
if e.crs.srs.lower() != 'epsg:2154':
|
||||
e.to_crs('epsg:2154',inplace=True)
|
||||
ef = (gpd.pd.concat([ef0,ef1,ef2,ef3,ef4])
|
||||
.astype({'code_ensem':str})
|
||||
.sort_values(['nom_ensemb'],ignore_index=True)
|
||||
.drop_duplicates(ignore_index=True))
|
||||
ef.to_file(path.join(PATH,'..','ensemblefonctionnelscomplte','Ensembles_Fonctionnels_Platière_Lambert93_ALL.geojson'),driver='GeoJSON')
|
||||
data = gpd.pd.read_csv(path.join(PATH,FILE_DATA),sep=";",encoding='Windows 1252') # ['ASCII','Windows 1252']
|
||||
|
||||
|
||||
dict_col = {
|
||||
**dict.fromkeys(['longitude autre'],'Long autre'),
|
||||
**dict.fromkeys(['latitude autre'],'Lat autre'),
|
||||
**dict.fromkeys([' Longitude grade ','LONGITUDE'],'Longitude grade'),
|
||||
**dict.fromkeys([' Latitude grade ','LATITUDE'],'Latitude grade'),
|
||||
**dict.fromkeys(['latitude autre'],'Lat autre'),
|
||||
**dict.fromkeys(['Numéro relevé','Numéro de relevé'],'Numéro'),
|
||||
}
|
||||
|
||||
dict_ef = {
|
||||
'Prairie ancienne pépinière CNR':'Ancienne pépinière CNR',
|
||||
'Marais méandre des Oves':'Marais des Oves',
|
||||
'Prairie des Iles (Péage)':'Prairie des Iles',
|
||||
'Vieux Rhône Péage amont seuil':'Vieux Rhône amont seuil Sablons',
|
||||
'Lône de la Platière':'Lone Platière',
|
||||
'Banc Arcoules (banc1)':'Banc Arcoule (banc 1)',
|
||||
'Plaine St Marice rive gauche canal':'Plaine St Maurice rive gauche canal',
|
||||
'Casiers île des Graviers':'Casiers aval observatoire',
|
||||
'Prairie violettes (vers Marais des Oves)':'Prairie Violettes',
|
||||
'Forêt nord des Oves':'Forêt Nord Oves',
|
||||
'Digue neuve et entonnement de la lône':'Digue neuve et entonnement',
|
||||
'Marais méandre des Oves':'Marais des Oves',
|
||||
'Banc et ilots aval seuil Peyraud':'banc et ilots aval seuil Peyraud',
|
||||
'Casiers banc 3':'casiers banc 3',
|
||||
'Casiers RD Limony/Serrières':'Casiers RD Limony Serrieres',
|
||||
'Forêt Pointe sud':'Forêt pointe Sud',
|
||||
'Gravière des Rotissots (CORA)':'Gravière des Rotissots(CORA)',
|
||||
'Lone des îles':'Boisements MICRONAT',
|
||||
"Lône de l'Ilon":'Lône Ilon',
|
||||
'Lône du Bayard (Donzère)':'Lône du Bayard',
|
||||
'Lône du Noyer nord':'Lône Noyer Nord',
|
||||
'Lône du Noyer sud':'Lône Noyer Sud',
|
||||
'Lone de la Roussette':'Lône de la Roussette',
|
||||
"Plan d'eau St Pierre de Bœuf":"Plan d'eau St Pierre de Boeuf",
|
||||
'Prairie SW +extS+ zone embroussaillée':'Prairie SW + ext Sud + zone embrousaillée',
|
||||
'Remblais Ecluse Sablons':'Remblais usine ecluse',
|
||||
'Roselière et marais St-Maurice':'Roselière et marais St Maurice',
|
||||
'Vieux Rhône Péage aval seuil Sablons':'Vieux Rhône aval Sablons',
|
||||
'Zone cultivée nord':'Zone cultivée Nord',
|
||||
'Contre canal RD canal Péage':'Contre-canal RD Péage',
|
||||
'Aval Ruisseau de Limony':'Foret Limony',
|
||||
'Forêts des grandes Oves':'Forêt des Grandes Oves',
|
||||
'Ile du Noyer nord':'Ile Noyer Nord',
|
||||
"Lone de l'ile de Peyraud":"Lône de l'ile de Peyraud",
|
||||
'Casiers banc 3':'casiers banc 3',
|
||||
'Foret Ile Bugnon':'Forêt Ile Bugnon',
|
||||
'Graviere des Rotissots (Commune)':'Gravière Rotissots (Commune)',
|
||||
'Plaine de St Pierre de Bœuf':'Plaine St Pierre de Boeuf',
|
||||
'Plaine nord Serrières':'Plaine Nord Serrières',
|
||||
'Zone cultivée sud':'Zone cultivée Sud',
|
||||
'Casiers Boussarde':'casiers Boussarde',
|
||||
'Casiers aval seuil Peyraud RG':'Casiers aval seil Peyraud RG',
|
||||
'Contre-canal rive gauche Péage':'Contre-canal RG Péage',
|
||||
'Ile du Noyer sud + boisement ouest':'Ile Noyer Sud + boisement ouest',
|
||||
'Lone de la Barcasse':'Lône de la Barcasse',
|
||||
'Ruisseau Dolon':'Ruisseau Sanne Dolon', # c'est bon !?
|
||||
'la Payre, cours aval + cc RD amont':'La Payre, cours aval + CC RD amont',
|
||||
'Lône de la quarantaine':'Lône de la Quarantaine',
|
||||
"Plan d'eau des Blaches":"plan d'eau des Blaches",
|
||||
'Lone du Gouvernement':'Lône du Gouvernement',
|
||||
'Anc. Gravière CNR Amont RCC Donzère':'Anc Gravière CNR Amont RCC', # ?
|
||||
'Zone réhabilitée traitement granulat RCC Baix':'Zone réhabilitée traitement granulat', # ?
|
||||
# 'Banc 1 RCC Donzère',
|
||||
# 'Butte (Arras sur Rhône)',
|
||||
# 'Canaux et ruisseaux Ancone',
|
||||
# 'Canaux et ruisseaux Petits Robins',
|
||||
# 'Casiers Ile du Chambon',
|
||||
# 'Etang de la Gare',
|
||||
# 'Forêt Limony (hors RN)',
|
||||
# 'Graviere Ancone',
|
||||
# 'Gravières Cruas',
|
||||
# 'Le Petit-Rhône de la Voulte/Livron',
|
||||
# 'Lone Ancone',
|
||||
# 'Prairie ile du Chambon',
|
||||
# 'Prairie sèche "Picard", St Gervais/Roubion',
|
||||
# 'Rivière Roubion',
|
||||
# 'Ruisseau de Limony (gorges)',
|
||||
# 'Ruisseau de la Riaille',
|
||||
# 'Ruisseau du Lumas',
|
||||
# 'Ruisseau ile du Chambon',
|
||||
|
||||
|
||||
# 'Casiers RCC Gervans':,
|
||||
# 'Forêt Limony (hors RN)':,
|
||||
# 'coteaux rive droite de St Pierre à Champagne':,
|
||||
# 'Butte (Arras sur Rhône)':,
|
||||
# 'Prairie ile du Chambon':,
|
||||
# 'Ravin de Serrières':,
|
||||
# 'ruisseau de Peyraud (gorges)':,
|
||||
# 'Rivière Roubion':'Roubion',
|
||||
# 'Banc 2 RCC Donzère',
|
||||
# 'Canaux et ruisseaux Petits Robins',
|
||||
# 'Contre canal RG Planaris',
|
||||
# 'Contre canaux Cruas',
|
||||
# 'Etang Chanson (Chavanay)',
|
||||
# 'Etangs Verlieux',
|
||||
# 'Forêt alluviale Roubion',
|
||||
# 'Grand val (Chavanay)',
|
||||
# 'Graviere Ancone',
|
||||
# 'Lac des Sauzets',
|
||||
# 'Le Petit-Rhône de la Voulte/Livron',
|
||||
# 'Lone des Petits Robins',
|
||||
# 'Marais "de Printegarde"',
|
||||
# "Plan d'eau loisir Le Pouzin",
|
||||
# 'Prairie sèche Pierrelatte',
|
||||
# 'Retenue de Baix-Logis Neuf',
|
||||
# 'Rivière Drôme aval ramières',
|
||||
# 'Rivière Roubion',
|
||||
# 'Ruisseau de Limony (gorges)',
|
||||
# 'Ruisseau ile du Chambon',
|
||||
# 'Vieux rhone de baix',
|
||||
# 'Vieux-Rhône Donzère --> confluent Ardèche',
|
||||
# 'ruisseau de Peyraud (gorges)',
|
||||
|
||||
}
|
||||
|
||||
|
||||
def remove_special_char(obj,space=False):
|
||||
dict_char = {
|
||||
r'[’]':"'",
|
||||
r'[àáâãäå]':'.',
|
||||
r'[èéêë]':'.',
|
||||
r'[ìíîï]':'.',
|
||||
r'[òóôõö]':'.',
|
||||
r'[ùúûü]':'.',
|
||||
r'[ -]':"",
|
||||
r'[–?|!^]':"."
|
||||
}
|
||||
if space:
|
||||
dict_char = {**dict_char, **{r'[ ]':""}}
|
||||
return obj.replace(dict_char,regex=True)
|
||||
|
||||
|
||||
|
||||
def exend_lines(line,geom):
|
||||
if line.geom_type == 'MultiLineString':
|
||||
mini = gpd.GeoSeries(list(line.geoms),crs=2154).distance(geom).min()
|
||||
line = [l for l in line.geoms if l.distance(geom) == mini][0]
|
||||
x,y = line.coords.xy
|
||||
ysup = y[-1] - y[0] if y[0] < y[-1] else y[0] - y[-1]
|
||||
xsup = x[-1] - x[0] if x[0] < x[-1] else x[0] - x[-1]
|
||||
x[0] = x[0] + xsup if x[0] > x[-1] else x[0] - xsup
|
||||
x[-1] = x[-1] - xsup if x[0] > x[-1] else x[-1] + xsup
|
||||
y[0] = y[0] + ysup if y[0] > y[-1] else y[0] - ysup
|
||||
y[-1] = y[-1] - ysup if y[0] > y[-1] else y[-1] + ysup
|
||||
return LineString(zip(x,y))
|
||||
|
||||
def test_data_ok(df,envelop):
|
||||
test_data_ok = gpd.GeoDataFrame(
|
||||
df,
|
||||
geometry=gpd.points_from_xy(df['Long autre'],df['Lat autre']),
|
||||
crs=27572
|
||||
).to_crs(epsg=2154)
|
||||
test_enveloppe = test_data_ok.intersects(envelop)
|
||||
is_ok = test_data_ok[test_enveloppe].copy()
|
||||
not_ok = test_data_ok[~test_enveloppe].copy()
|
||||
not_ok['Lat autre'] = None
|
||||
not_ok['Long autre'] = None
|
||||
return is_ok.drop('geometry',axis=1),not_ok.drop('geometry',axis=1)
|
||||
|
||||
|
||||
def get_nearest_values(row, other_gdf, grid, point_column='geometry'):
|
||||
"""Find the nearest point and return the corresponding value from specified value column."""
|
||||
|
||||
# Recréation de la maille
|
||||
lonG = row['Latitude grade']
|
||||
latG = row['Longitude grade']
|
||||
long = grid[grid.Champ1 == lonG].unary_union
|
||||
lat = grid[grid.Champ1 == latG].unary_union
|
||||
longSup = grid[grid.Champ1 == round((lonG+0.0025),4)].unary_union
|
||||
latSup = grid[grid.Champ1 == round((latG+0.0025),4)].unary_union
|
||||
|
||||
if longSup is None:
|
||||
longInf = grid[grid.Champ1 == round((lonG-0.0025),4)].unary_union
|
||||
d = long.distance(longInf)
|
||||
x,y = long.xy
|
||||
y[0] = y[0] + d
|
||||
y[-1] = y[-1] + d
|
||||
longSup = LineString(zip(x,y))
|
||||
|
||||
|
||||
p1 = long.intersection(lat)
|
||||
while p1.is_empty:
|
||||
long = exend_lines(long,lat)
|
||||
lat = exend_lines(lat,long)
|
||||
p1 = long.intersection(lat)
|
||||
|
||||
p2 = longSup.intersection(lat)
|
||||
while p2.is_empty:
|
||||
longSup = exend_lines(longSup,lat)
|
||||
lat = exend_lines(lat,longSup)
|
||||
p2 = longSup.intersection(lat)
|
||||
|
||||
p3 = longSup.intersection(latSup)
|
||||
while p3.is_empty:
|
||||
longSup = exend_lines(longSup,latSup)
|
||||
latSup = exend_lines(latSup,longSup)
|
||||
p3 = longSup.intersection(latSup)
|
||||
|
||||
p4 = long.intersection(latSup)
|
||||
while p4.is_empty:
|
||||
long = exend_lines(long,latSup)
|
||||
latSup = exend_lines(latSup,long)
|
||||
p4 = long.intersection(latSup)
|
||||
|
||||
pointList = [p1, p2, p3, p4, p1]
|
||||
for i,p in enumerate(pointList):
|
||||
if p.geom_type == 'MultiPoint':
|
||||
for pp in list(p.geoms):
|
||||
if p.distance(row.geom) == row.geom.distance(pp):
|
||||
pointList[i] = pp
|
||||
poly = gpd.GeoSeries(Polygon([[p.x, p.y] for p in pointList]).buffer(0),crs=2154)
|
||||
|
||||
# Filtre de l'ensemble fonctionnel
|
||||
other_geom = other_gdf.geometry.name
|
||||
other_gdf_filter = other_gdf[other_gdf.nom_ensemb == row.esmb].copy()
|
||||
|
||||
if other_gdf_filter.contains(row.geom).any():
|
||||
# pass
|
||||
return row.geom
|
||||
elif other_gdf_filter.intersection(poly.unary_union).is_empty.all():
|
||||
pass
|
||||
else:
|
||||
other_gdf_filter[other_geom] = other_gdf_filter.intersection(poly.unary_union)
|
||||
|
||||
# Create an union of the other GeoDataFrame's geometries:
|
||||
other_points = other_gdf_filter[other_geom].unary_union
|
||||
|
||||
# Find the nearest points
|
||||
nearest_geoms = nearest_points(row[point_column], other_points)
|
||||
|
||||
# Get corresponding values from the other df
|
||||
# nearest_data = other_gdf.loc[other_gdf[other_geom] == nearest_geoms[1]]
|
||||
|
||||
# nearest_value = nearest_data[value_column].get_values()[0]
|
||||
|
||||
return nearest_geoms[1]
|
||||
|
||||
def create_grid():
|
||||
|
||||
grid = gpd.read_file(path.join(PATH,'..',FILE_GRID))
|
||||
grid.rename(columns={'Coordonné':'Champ1'},inplace=True)
|
||||
if grid.crs.srs.lower()!='epsg:2154':
|
||||
grid.to_crs(epsg=2154,inplace=True)
|
||||
if grid['Champ1'].dtype == object:
|
||||
grid['Champ1'] = grid['Champ1'].str.replace(',','.').astype(float)
|
||||
grid = grid.dissolve(by='Champ1',as_index=False)
|
||||
|
||||
|
||||
# Filtre Lat/Long
|
||||
long = grid[grid.Champ1 < 25].copy()
|
||||
lat = grid[grid.Champ1 > 25].copy()
|
||||
|
||||
# Identification du point d'intersection
|
||||
grd = gpd.GeoSeries(list(long.unary_union.intersection(lat.unary_union).geoms),crs=2154)\
|
||||
.to_frame()\
|
||||
.rename_geometry('geom')
|
||||
# Récuperation des coordonnées x et y
|
||||
grd['x'] = grd.geom.x.copy()
|
||||
grd['y'] = grd.geom.y.copy()
|
||||
grd = grd.sort_values(['y','x'],ascending=[True,False]).reset_index(drop=True)
|
||||
x0 = (grd.x[1]-grd.x[0])/2
|
||||
y0 = (grd.y[1]-grd.y[0])/2
|
||||
grd = grd.sort_values(['x','y'],ascending=[True,False]).reset_index(drop=True)
|
||||
x1 = (grd.x[1]-grd.x[0])/2
|
||||
y1 = (grd.y[0]-grd.y[1])/2
|
||||
X = x0+x1
|
||||
Y = y0+y1
|
||||
return grid,X,Y
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
|
||||
lst_file = listdir(PATH)
|
||||
lst_file.sort()
|
||||
|
||||
for FILE_DATA in lst_file[22:]:
|
||||
|
||||
grid,X,Y = create_grid()
|
||||
envlop = grid.geometry.unary_union.envelope.buffer(10000)
|
||||
data = gpd.pd.read_csv(path.join(PATH,FILE_DATA),sep=";",encoding='Windows 1252') # ['ASCII','Windows 1252']
|
||||
data.dropna(how='all',inplace=True)
|
||||
data.rename(columns=dict_col,inplace=True)
|
||||
|
||||
print('File :',FILE_DATA)
|
||||
# Si Long/Lat est un champ text, transformation des colonnes en nombre.
|
||||
if ('Long autre' in data.columns) :
|
||||
if (data['Long autre'].dtype == object):
|
||||
data['Long autre'] = data['Long autre'].str.replace(',','.').astype(float)
|
||||
if ('Lat autre' in data.columns) :
|
||||
if data['Lat autre'].dtype == object:
|
||||
data['Lat autre'] = data['Lat autre'].str.replace(',','.').astype(float)
|
||||
if data['Longitude grade'].dtype == object:
|
||||
data['Longitude grade'] = data['Longitude grade'].str.replace(',','.').astype(float)
|
||||
if data['Latitude grade'].dtype == object:
|
||||
data['Latitude grade'] = data['Latitude grade'].str.replace(',','.').astype(float)
|
||||
|
||||
data.loc[data['Lat autre']==0,'Lat autre'] = None
|
||||
data.loc[data['Long autre']==0,'Long autre'] = None
|
||||
|
||||
if not ef.is_valid.all():
|
||||
ef.geometry = ef.make_valid()
|
||||
|
||||
# Isolement des données précises
|
||||
data_ok = data.loc[
|
||||
data['Long autre'].notna() & data['Lat autre'].notna()
|
||||
].copy()
|
||||
|
||||
if data.shape[0] == data_ok.shape[0]:
|
||||
continue
|
||||
|
||||
data_ok, data_not_ok = test_data_ok(data_ok,envlop)
|
||||
|
||||
df = gpd.pd.concat([
|
||||
data.loc[
|
||||
data['Long autre'].isna()&data['Lat autre'].isna()
|
||||
].copy(),
|
||||
data_not_ok]
|
||||
).sort_values('Numéro').reset_index() # 'Numero'
|
||||
|
||||
if data.shape[0] == data_ok.shape[0]:
|
||||
continue
|
||||
|
||||
# Data to GéoData
|
||||
gdf_ok = gpd.GeoDataFrame(
|
||||
data_ok,
|
||||
geometry=gpd.points_from_xy(data_ok['Long autre'],data_ok['Lat autre']),
|
||||
crs=27572
|
||||
).to_crs(epsg=2154)
|
||||
|
||||
# Correspondance [ Grades - Grid ]
|
||||
DICT_GRID = dict(zip(grid.Champ1,grid.geometry.to_wkt()))
|
||||
# Replace Grade by L93
|
||||
df[['Lat93','Long93']] = df[['Latitude grade','Longitude grade']].replace(DICT_GRID)
|
||||
|
||||
# Reconstruction du GéoDataframe
|
||||
gdf = gpd.GeoDataFrame(
|
||||
df.copy(),
|
||||
geometry=gpd.GeoSeries.from_wkt(df['Lat93'])\
|
||||
.intersection(gpd.GeoSeries.from_wkt(df['Long93'])),
|
||||
crs=2154
|
||||
)
|
||||
|
||||
geom_empty = gdf.geometry.is_empty
|
||||
if geom_empty.any():
|
||||
for i,row in gdf.loc[geom_empty].iterrows():
|
||||
while wkt.loads(row['Lat93']).intersection(wkt.loads(row['Long93'])).is_empty:
|
||||
row['Lat93'] = exend_lines(
|
||||
wkt.loads(row['Lat93']),
|
||||
wkt.loads(row['Long93'])
|
||||
).wkt
|
||||
row['Long93'] = exend_lines(
|
||||
wkt.loads(row['Long93']),
|
||||
wkt.loads(row['Lat93'])
|
||||
).wkt
|
||||
gdf.loc[i,'geometry'] = wkt.loads(row['Lat93']).intersection(wkt.loads(row['Long93']))
|
||||
|
||||
gdf.rename_geometry('geom',inplace=True)
|
||||
gdf['x'] = gdf.geom.x + X
|
||||
gdf['y'] = gdf.geom.y + Y
|
||||
gdf.set_geometry(gpd.points_from_xy(gdf.x,gdf.y),inplace=True)
|
||||
|
||||
# Test
|
||||
ef_exp = ef.explode(index_parts=True)
|
||||
# [
|
||||
# near(geom,ldt,ldt_union,'id_lieu_dit')
|
||||
# for geom in df.geometrie
|
||||
# ]
|
||||
|
||||
# gdf['Code ensemble fonct']
|
||||
|
||||
gdf['esmb'] = remove_special_char(
|
||||
gdf['Ensemble fonctionnel'].replace(dict_ef)
|
||||
)
|
||||
ef_exp.nom_ensemb = remove_special_char(ef_exp.nom_ensemb)
|
||||
|
||||
gdf['nearest_point'] = gdf.apply(get_nearest_values, other_gdf=ef_exp, grid=grid, point_column="geom", axis=1)
|
||||
gdf.drop(['geom','esmb'],axis=1,inplace=True)
|
||||
gdf.set_geometry('nearest_point',crs=2154,inplace=True)
|
||||
gdf.rename_geometry('geometry',inplace=True)
|
||||
|
||||
if not gdf_ok.empty:
|
||||
gdf = gpd.pd.concat([gdf,gdf_ok])
|
||||
|
||||
FILE_OUT = FILE_DATA.replace('.csv','_reprojected.csv')
|
||||
gdf.to_csv(path.join(PATH,'..','RESULT',FILE_OUT),sep=';',encoding='Windows 1252',index=False)
|
||||
|
||||
FILE_OUT = FILE_OUT.replace('-','_').replace(' ','_')
|
||||
FILE_OUT = FILE_OUT.replace('(','').replace(')','')
|
||||
(gdf
|
||||
.to_file(path.join(PATH,'..','RESULT','result_2.gpkg'),driver='GPKG',layer=FILE_OUT.replace('-','_')))
|
||||
@@ -0,0 +1,80 @@
|
||||
from sqlalchemy import create_engine # pour lecture de la bd Géonature
|
||||
from sqlalchemy.engine import URL
|
||||
from pycen import update_to_sql
|
||||
import pandas as pd
|
||||
|
||||
usr = 'postgres'
|
||||
pdw = 'postgres'
|
||||
bdd = 'serenadb'
|
||||
host = '172.17.0.2'
|
||||
|
||||
eng = URL.create('postgresql+psycopg2',username=usr,password=pdw,host=host,database=bdd)
|
||||
conn = create_engine(eng)
|
||||
|
||||
obs = pd.read_sql_table('rnf_obse',con=conn,schema='serenabase')
|
||||
|
||||
|
||||
|
||||
## Foreign keys table : serenabase.rnf_choi
|
||||
id_cols = obs.columns[obs.columns.str.endswith('_choi_id')]
|
||||
|
||||
update_to_sql(obs[['obse_id',*id_cols]].replace(0,None).astype('Int64'),
|
||||
conn,'rnf_obse','serenabase','obse_id')
|
||||
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_sexe_fk FOREIGN KEY (obse_sex_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_stade_fk FOREIGN KEY (obse_stade_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_ageunit_fk FOREIGN KEY (obse_ageunit_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_abond_fk FOREIGN KEY (obse_abond_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_precis_fk FOREIGN KEY (obse_precis_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_soci_fk FOREIGN KEY (obse_soci_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_comp_fk FOREIGN KEY (obse_comp_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_compdir_fk FOREIGN KEY (obse_compdir_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_contact_fk FOREIGN KEY (obse_contact_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_contact2_fk FOREIGN KEY (obse_contact2_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_activ_fk FOREIGN KEY (obse_activ_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_caract_fk FOREIGN KEY (obse_caract_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_etat_fk FOREIGN KEY (obse_etat_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_derango_fk FOREIGN KEY (obse_derango_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_derangi_fk FOREIGN KEY (obse_derangi_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_nebul_fk FOREIGN KEY (obse_nebul_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_precip_fk FOREIGN KEY (obse_precip_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_vent_fk FOREIGN KEY (obse_vent_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_ventdir_fk FOREIGN KEY (obse_ventdir_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_visib_fk FOREIGN KEY (obse_visib_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_sol_fk FOREIGN KEY (obse_sol_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_eau_fk FOREIGN KEY (obse_eau_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_choi_eaudir_fk FOREIGN KEY (obse_eaudir_choi_id) REFERENCES serenabase.rnf_choi(choi_id);')
|
||||
|
||||
|
||||
id_cols = obs.columns[obs.columns.str.endswith('user_id')]
|
||||
with conn.begin() as cnx:
|
||||
cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_user_crea_fk FOREIGN KEY (obse_crea_user_id) REFERENCES serenabase.rnf_user(user_id);')
|
||||
|
||||
# DETAIL: La clé (obse_lmod_user_id)=(bbb76b84-493a-4d26-8f01-0440d377198b) n'est pas présente dans la table « rnf_user ».
|
||||
# with conn.begin() as cnx:
|
||||
# cnx.execute('ALTER TABLE serenabase.rnf_obse ADD CONSTRAINT rnf_obse_rnf_user_lmod_fk FOREIGN KEY (obse_lmod_user_id) REFERENCES serenabase.rnf_user(user_id);')
|
||||
|
||||
id_cols = obs.columns[(obs.columns.str.endswith('_id'))& (~obs.columns.str.contains('choi'))]
|
||||
@@ -0,0 +1,208 @@
|
||||
from pycen import con_sicen, update_to_sql
|
||||
from sqlalchemy import create_engine # pour lecture de la bd Géonature
|
||||
from sqlalchemy.engine import URL
|
||||
from shapely.geometry import Polygon
|
||||
import geopandas as gpd
|
||||
import pandas as pd
|
||||
import json
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
PATH = '/home/colas/Documents/9_PROJETS/6_GEONATURE/MIGRATION/RNGL/DUMP/'
|
||||
FILE = 'export_rnngl.txt'
|
||||
|
||||
usr = 'postgres'
|
||||
pdw = 'postgres'
|
||||
bdd = 'serenadb'
|
||||
host = '172.17.0.2'
|
||||
|
||||
eng = URL.create('postgresql+psycopg2',username=usr,password=pdw,host=host,database=bdd)
|
||||
conn = create_engine(eng)
|
||||
|
||||
tax = pd.read_sql_table('rnf_taxo',con=conn,schema='serenarefe')
|
||||
obs = pd.read_sql_table('rnf_obse',con=conn,schema='serenabase')
|
||||
sit = pd.read_sql_table('rnf_site',con=conn,schema='serenabase')
|
||||
exp = pd.read_table(PATH+FILE,sep='\t',encoding='Windows-1252',dtype={'OBSE_NOMBRE':str,'OBSE_ABOND_CHOI':str,'OBSE_DATE':str})
|
||||
exp.columns = exp.columns.str.lower()
|
||||
exp[['obse_lat','obse_lon','obse_nombre','oglat','oglon']] = exp[['obse_lat','obse_lon','obse_nombre','oglat','oglon']].replace(',','.',regex=True)
|
||||
cols_coord = exp.columns[exp.columns.str.endswith('lat|lon')]
|
||||
year_only = exp.obse_date.str.len() == 4
|
||||
month_only = exp.obse_date.str.len() == 6
|
||||
exp.loc[year_only,'obse_date'] = exp.loc[year_only,'obse_date'] + '0101'
|
||||
exp.loc[month_only,'obse_date'] = exp.loc[month_only,'obse_date'] + '01'
|
||||
exp = (gpd.GeoDataFrame(
|
||||
exp,crs=4326,
|
||||
geometry=gpd.points_from_xy(exp.oglon,exp.oglat))
|
||||
.to_crs(2154))
|
||||
exp['obse_date'] = pd.to_datetime(exp.obse_date)
|
||||
|
||||
# Re-création des géométries des sites
|
||||
sit['poly'] = (sit
|
||||
.site_poly.str[9:]
|
||||
.str.split(','))
|
||||
|
||||
sit.loc[sit.poly.notna(),'geom'] = (sit.loc[sit.poly.notna(),'poly']
|
||||
.apply(lambda x: [xx.split(' ') for xx in x if xx])
|
||||
.apply(lambda x: [[float(xxx) for xxx in xx] for xx in x ])
|
||||
.apply(lambda x: Polygon(x)))
|
||||
|
||||
sit = (sit
|
||||
.set_geometry('geom',crs=4326)
|
||||
.to_crs(2154))
|
||||
|
||||
# Création des géométries à partir des coordonnées
|
||||
geom_na = obs.geom_obse.isna()
|
||||
idx = obs[geom_na].obse_id.tolist()
|
||||
|
||||
# Test des données de l'export
|
||||
exp[exp.obse_id.isin(idx)]
|
||||
|
||||
|
||||
# Mise à jour des observations-sites
|
||||
obs_sit = (obs[geom_na]
|
||||
.merge(sit[['site_id','geom']],how='left',left_on='obse_site_id',right_on='site_id')
|
||||
.set_geometry('geom',crs=2154))
|
||||
update_to_sql(
|
||||
obs_sit[['obse_id','geom']].rename_geometry('geom_site'),
|
||||
conn,'rnf_obse','serenabase','obse_id')
|
||||
|
||||
#### IDENTIFICATION SERENA-GLEMPS ON SICEN ####
|
||||
|
||||
|
||||
#### IDENTIFICATION SERENA-GLEMPS ON SICEN ####
|
||||
obs = pd.read_sql_table('rnf_obse',con=conn,schema='serenabase')
|
||||
sic = pd.read_sql_table('saisie_observation',con=con_sicen,schema='saisie')
|
||||
vsi = pd.read_sql_table('v_saisie_observation',con=con_sicen,schema='saisie')
|
||||
# vmsi = pd.read_sql_table('vm_synthese_observations',con=con_sicen,schema='saisie')
|
||||
vmsi = gpd.read_postgis('SELECT * FROM saisie.vm_synthese_observations',con_sicen,crs=2154)
|
||||
vmsi['date_obs'] = pd.to_datetime(vmsi.date_obs)
|
||||
test_obsEXP = exp.obsv_libel.str.contains('BIRON|JUTON|MARRON',na=False)
|
||||
exp[test_obsEXP]
|
||||
|
||||
id_obsGL = obs.obse_id.astype(str).tolist()
|
||||
t1_obsVM = ~vmsi.observateurs_v2.str.contains('BIRON',na=False)
|
||||
t2_obsVM = vmsi.rmq_localisation.str.contains('GRAND-LEMPS|GRAND LEMPS',na=False,case=False)
|
||||
t3_obsVM = ~vmsi.id_origine.isin(exp[test_obsEXP].obse_id.astype(str))
|
||||
t4_obsVM = vmsi.date_obs <= '2019-01-01'
|
||||
t5_obsVM = vmsi.etude != 'Import données Pro Faune Isère'
|
||||
t6_obsVM = vmsi.id_origine.isin(id_obsGL)
|
||||
|
||||
GL = vmsi[t2_obsVM&t1_obsVM].copy()
|
||||
GL['lat'] = GL.to_crs(4326).geometry.y.round(10).astype(str)
|
||||
GL['lon'] = GL.to_crs(4326).geometry.x.round(10).astype(str)
|
||||
|
||||
while (GL.lat.str.len()==13).all() == False:
|
||||
GL.loc[GL.lat.str.len()<13,'lat'] = GL[GL.lat.str.len()<13].lat + '0'
|
||||
|
||||
while (GL.lon.str.len()==12).all() == False:
|
||||
GL.loc[GL.lon.str.len()<12,'lon'] = GL[GL.lon.str.len()<12].lon + '0'
|
||||
|
||||
test = GL[['lat','lon']].eq(exp[['oglat','oglon']])
|
||||
test = GL.intersects(exp.unary_union)
|
||||
GL[test]
|
||||
# 18819 Données dans Sicen Qui concerne le Grand-Lemps
|
||||
data_gl = vmsi[t6_obsVM]
|
||||
t7 = ~vmsi.id_obs.isin(data_gl.id_obs)
|
||||
GL = vmsi[t1_obsVM&t2_obsVM&t4_obsVM&t5_obsVM&t7].copy()
|
||||
4259862565
|
||||
45.4259862565
|
||||
|
||||
id_obs = obs.obse_id.astype(str).tolist()
|
||||
vsi[vsi.id_origine.isin(id_obs)].astype({'id_origine':int}).sort_values('id_origine')
|
||||
vsi[vsi.id_origine.isin(id_obs)].date_obs.max()
|
||||
|
||||
loc_glps = vsi.localisation.str.contains('ETANG DU GRAND-LEMPS',na=False)
|
||||
vsi[loc_glps].observateur.unique()
|
||||
loc_glps = vmsi.rmq_localisation.str.contains('ETANG DU GRAND-LEMPS',na=False)
|
||||
loc_avenir = vmsi.observateurs_v2.str.contains('BIRON|JUTON|MARRON',na=False)
|
||||
vmsi[loc_glps&~loc_avenir].observateurs_v2.unique()
|
||||
(vsi[loc_glps].observateur
|
||||
.astype(list))
|
||||
|
||||
|
||||
## By OBSV_LIBEL + DATE
|
||||
rnn = exp.copy()
|
||||
rnn.obsv_libel = (rnn.obsv_libel
|
||||
.replace(r'\(.*?\)','',regex=True)
|
||||
.str.strip()
|
||||
.replace({' - ':' '},regex=True)
|
||||
.replace({' ':' '},regex=True)
|
||||
.replace({' et ':' & '},regex=True)
|
||||
.replace({'&':','},regex=True)
|
||||
.replace({' , ':', '},regex=True)
|
||||
.replace({' , ':', '},regex=True)
|
||||
.apply(lambda x: ', '.join(sorted(x.split(', ')))))
|
||||
|
||||
sieg = vmsi.copy()
|
||||
sieg.observateurs_v2 = (sieg.observateurs_v2
|
||||
.replace(r'\(.*?\)','',regex=True)
|
||||
.str.strip()
|
||||
.replace({' ':' '},regex=True)
|
||||
.replace({' ,':', '},regex=True)
|
||||
.replace({' ,':', '},regex=True)
|
||||
.apply(lambda x: ', '.join(sorted(x.split(', ')))))
|
||||
|
||||
sit.set_index('site_id',inplace=True)
|
||||
|
||||
lst_etude_serena = [
|
||||
'Protocole de suivi AVENIR',
|
||||
'Veille écologique AVENIR',
|
||||
'Anciennes BD AVENIR',
|
||||
'BIBLIOGRAPHIE',
|
||||
'Échange de données - CORA Rhône',
|
||||
'Veille écologique',
|
||||
'Échange de données - CBNA',
|
||||
'Echange de données - GRPLS',
|
||||
# 'Echange de données (partenaires)'
|
||||
]
|
||||
t1 = sieg.observateurs_v2.isin(rnn.obsv_libel)
|
||||
t2 = sieg.date_obs.isin(rnn.obse_date)
|
||||
t3 = sieg.intersects(sit.iloc[2].geom.buffer(0))
|
||||
t4 = sieg.etude.isin(lst_etude_serena)
|
||||
t4 = sieg.etude=='Échange de données - CORA Rhône'
|
||||
sieg[t1&t2&t3&t4] # Données à écarter. 15212 données dans Sicen qui concerne le Grand-Lemps
|
||||
# 10404 dans le Grand-Lemps avant le 01-01-2018
|
||||
|
||||
date_max = sieg[t1&t2&t3&t4].date_obs.max()
|
||||
rnn[rnn.obse_date <= date_max]
|
||||
t5 = sieg.date_obs==date_max
|
||||
sieg[t1&t2&t3&t4&(~t5)]
|
||||
dat_max2 = sieg[t1&t2&t3&t4&(~t5)].date_obs.max()
|
||||
t6 = sieg.date_obs==dat_max2
|
||||
sieg[t1&t2&t3&t4&(~t5)&(~t6)]
|
||||
sieg[t1&t2&t3&t4&t6]
|
||||
|
||||
dict_col = {
|
||||
'observateurs_v2':'obsv_libel',
|
||||
'date_obs':'obse_date',
|
||||
# 'lot_donnee':'lot_donnee',
|
||||
'etude':'relv_nom',
|
||||
'nom_complet':'taxo_latin_c',
|
||||
# 'geom':'geometry',
|
||||
}
|
||||
fusion = pd.concat(
|
||||
[sieg[t1&t2&t3&t4].rename(columns=dict_col)[[*dict_col.values()]],
|
||||
rnn[[*dict_col.values()]]],
|
||||
)
|
||||
|
||||
|
||||
#### Test equivalence [obse_lat - oglat, obse_lon - oglon]
|
||||
# Columns :
|
||||
# obsv_libel : Observateur
|
||||
# obse_date : date d'Observation
|
||||
|
||||
test = exp[exp.obse_lat.notna()]
|
||||
tlat = test.obse_lat.eq(test.oglat)
|
||||
tlon = test.obse_lon.eq(test.oglon)
|
||||
test[~tlat].index.equals(test[~tlon].index)
|
||||
test[~tlat]
|
||||
test[~tlon]
|
||||
|
||||
lst_tax = tax.taxo_id.tolist()
|
||||
test = obs[~obs.obse_taxo_id.isin(lst_tax)]
|
||||
|
||||
# ERREUR: une instruction insert ou update sur la table « rnf_obse » viole la contrainte de clé
|
||||
# étrangère « rnf_obse_rnf_user_fk »
|
||||
# Détail : La clé (obse_obsv_id)=(6052) n'est pas présente dans la table « rnf_user ».
|
||||
test = ~vmsi.observateurs_v2.str.contains('Isère|Personnel',na=False,case=False)
|
||||
vmsi[test]
|
||||
vmsi[test].observateurs_v2
|
||||
@@ -0,0 +1,104 @@
|
||||
from pycen import con_gn
|
||||
import pandas as pd
|
||||
import os
|
||||
|
||||
|
||||
def get_synthese(con,lst):
|
||||
sql = '''
|
||||
SELECT * FROM gn_synthese.v_synthese_for_export
|
||||
WHERE cd_nom IN {lst_cdnom}
|
||||
'''.format(lst_cdnom=tuple(lst))
|
||||
return pd.read_sql_query(sql,con)
|
||||
|
||||
if __name__ == "__main__":
|
||||
PATH = '/media/cgeier/SRV/homer_geonat/home/geonatureadmin/taxhub/tmp'
|
||||
file = 'liste_changements.csv'
|
||||
chgt = pd.read_csv(os.path.join(PATH,file))
|
||||
|
||||
updt_cdnom = chgt.grappe_change=='cas1'
|
||||
cd_change = (chgt[~updt_cdnom].i_cd_nom_list
|
||||
.str.removeprefix('[')
|
||||
.str.removesuffix(']')
|
||||
.str.split(', ')
|
||||
.explode()
|
||||
.unique())
|
||||
df = get_synthese(con_gn,cd_change)
|
||||
|
||||
test = df.cd_nom == 134367
|
||||
df[test]
|
||||
df[test].observateurs
|
||||
df[test].nom_valide
|
||||
df[test].nom_vernaculaire
|
||||
|
||||
######
|
||||
### MODIFICATION A APPORTER DANS LA BDD AVANT EVOLUTION DU REFERENTIEL
|
||||
######
|
||||
|
||||
# 100142 OK - Reste inchangé, synonyme devient sous-espèce
|
||||
# 135069 Devient sous-espèce de 100142 49 Observations - GUEDOU Alix (CENI) & GOURGUES Frédéric (Gentiana) & ...
|
||||
### [Geranium robertianum L., 1753 --> Geranium robertianum subsp. robertianum L., 1753]
|
||||
### Mise à jour des obs 135069 --> 100142 Geranium robertianum L., 1753
|
||||
|
||||
# 126650 OK - Reste inchangé, synonyme 141769 devient sous-espèce
|
||||
# 141769 Devient sous-espèce de 126650 1 Observation 2008/10/30 - Raspail Loïc (AAIP)
|
||||
### [Tilia platyphyllos Scop., 1771 --> Tilia platyphyllos subsp. pseudorubra C.K.Schneid., 1909]
|
||||
### Mise à jour des obs 141769 --> 126650 Tilia platyphyllos
|
||||
### FAIT (modification dans l'import source)
|
||||
|
||||
######
|
||||
### LAISSER SUIVRE SON COURS
|
||||
######
|
||||
|
||||
# FLORE
|
||||
########
|
||||
# 145374 OK - Reste var. CD_REF, synonyme 619051 devient synonyme espèce. 619051 Non observé.
|
||||
# 33901 OK - Reste CD_REF, 33781 devient synonymes. 33781 Non observé.
|
||||
# 465265 OK - Reste synonyme de 966674. 966675 devient 2e synonyme. 966675 Non observé.
|
||||
# 53451 OK - Reste CD_REF, 9 synonymes deviennent espèces. Synonymes Non observé.
|
||||
# 89840 OK - Reste CD_REF, 619051 devient synonymes. 619051 Non observé.
|
||||
|
||||
# 108770 Devient synonyme de 762076 4 Observations - BIRON Nicolas (CENI) & GRANGE Benjamin (CENI)
|
||||
### [Monotropa hypopitys L., 1753 --> Hypopitys monotropa Crantz, 1766]
|
||||
# Vérifier espèce présente isère .. Oui
|
||||
|
||||
# 113620 OK - Reste CD_REF, taxon 138815,923321 deviennent synonymes
|
||||
# 138815 Regroupé avec 113620 11 Observations - BIRON Nicolas (CENI)
|
||||
### [Pinguicula grandiflora subsp. grandiflora Lam., 1789 --> Pinguicula grandiflora Lam., 1789]
|
||||
# Regarder champs remarque si pas tagué rosea .. Non
|
||||
# Vérifier espèce présente isère .. Oui
|
||||
|
||||
# 116759 OK - Reste CD_REF, taxon 139596 devient synonyme
|
||||
# 139596 Regroupé avec 116759 1 Observation 2004/12/01 - GILIOTTI Sébastien (Liparis SG)
|
||||
### [Quercus robur var. robur L., 1753 --> Quercus robur L., 1753]
|
||||
|
||||
# 97508 OK - Reste CD_REF, taxon 134367 devient synonyme
|
||||
# 134367 Regroupé avec 97508 1 Observation 2018/07/11 - Pont Bernard (AAIP)
|
||||
### [Euphorbia esula subsp. esula L., 1753 --> Euphorbia esula L., 1753]
|
||||
### Anomalie - modification dans l'import source
|
||||
|
||||
# FONGE
|
||||
########
|
||||
# 814216 OK - Reste CD_REF, 465268 devient synonymes
|
||||
# 465268 Devient synonyme de 814216 3 Observation 2015/xx/xx - RIVOIRE Bernard (MYCOPOLYDEV)
|
||||
### [Porodaedalea conchata (Pers.) Fiasson & Niemelä, 1984 --> Phellinopsis conchata (Pers.) Y.C.Dai, 2010]
|
||||
|
||||
# 44992 OK - Reste CD_REF, 470268 devient synonymes
|
||||
# 470268 Devient synonyme de 44992 2 Observation 2015/xx/xx - RIVOIRE Bernard (MYCOPOLYDEV)
|
||||
### [Tremella mesenterica f. crystallina Ew.Gerhardt, 1997 --> Tremella mesenterica Retz., 1769]
|
||||
|
||||
|
||||
lstjdd_plt = [
|
||||
'[CNR Restauration] Prospection flore vasculaire',
|
||||
'[ENS méandre des Oves] Observation opportuniste par observation directe',
|
||||
'[MC CNR Inspira] Inventaire dendrométrique + DMH',
|
||||
'[MC CNR Inspira] Relevés phytosociologiques et sondages pédologiques',
|
||||
'[N2000 Platière] Inventaire dendrométrique',
|
||||
'[RNN Platière] Observation opportuniste par observation directe',
|
||||
'[RNN Platière] Inventaire dendrométrique en plein',
|
||||
'[RNN Platière] Suivi phytosciologique grèves alluviales',
|
||||
'[RNN Platière] Prospection flore vasculaire',
|
||||
'[CNPE] Suivi phytosciologique des milieux ouverts',
|
||||
'[CNPE] Observation opportuniste par observation directe',
|
||||
'[ABC St-Maurice] Observation Opportuniste par observation directe'
|
||||
]
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
from pycen import con_gn
|
||||
import geopandas as gpd
|
||||
import requests
|
||||
|
||||
def get_list_observateur(code_liste,with_data=False):
|
||||
"""
|
||||
@code_liste : str. Nom du sous-module monitoring où chercher les sites.
|
||||
@with_data : bool.
|
||||
"""
|
||||
|
||||
if with_data is True:
|
||||
sql = '''
|
||||
SELECT * FROM utilisateurs.t_listes
|
||||
JOIN (utilisateurs.cor_role_liste JOIN utilisateurs.t_roles t USING (id_role)) USING(id_liste)
|
||||
'''
|
||||
elif with_data is False:
|
||||
sql = '''
|
||||
SELECT code_liste list_name,id_role,nom_role||' '||prenom_role nom_complet FROM utilisateurs.t_listes
|
||||
JOIN (utilisateurs.cor_role_liste JOIN utilisateurs.t_roles t USING (id_role)) USING(id_liste)
|
||||
'''
|
||||
else :
|
||||
raise('Argument `with_data` dont accept value "%s"'%with_data)
|
||||
|
||||
wh = " WHERE (id_liste = {mod} OR code_liste = '{mod}')".format(mod=code_liste)
|
||||
|
||||
SQL = sql + wh
|
||||
return gpd.pd.read_sql_query(sql=SQL,con=con_gn)
|
||||
|
||||
|
||||
|
||||
def get_site(module,id_site=None,with_data=False):
|
||||
"""
|
||||
@module : str. Nom du sous-module monitoring où chercher les sites.
|
||||
@id_site : str. Nom du site au sein du sous-monitoring où chercher les sites.
|
||||
@with_data : bool.
|
||||
"""
|
||||
|
||||
if with_data is True:
|
||||
sql = '''
|
||||
SELECT * FROM gn_monitoring.t_base_sites
|
||||
JOIN (gn_monitoring.cor_site_module JOIN gn_commons.t_modules t USING (id_module)) USING (id_base_site)
|
||||
'''
|
||||
elif with_data is False:
|
||||
sql = '''
|
||||
SELECT id_base_site,base_site_name,
|
||||
(ST_Y(geom)::numeric)::text||' '||(ST_X(geom)::numeric)::text geometry
|
||||
FROM gn_monitoring.t_base_sites
|
||||
JOIN (gn_monitoring.cor_site_module JOIN gn_commons.t_modules t USING (id_module)) USING (id_base_site)
|
||||
'''
|
||||
else :
|
||||
raise('Argument `with_data` dont accept value "%s"'%with_data)
|
||||
|
||||
wh = " WHERE (t.module_code = '{mod}' OR t.module_label = '{mod}')".format(mod=module)
|
||||
|
||||
if id_site is not None:
|
||||
wh = wh + " AND id_site = '%s'"%id_site
|
||||
|
||||
SQL = sql + wh
|
||||
return gpd.pd.read_sql_query(sql=SQL,con=con_gn)
|
||||
# return gpd.read_postgis(sql=SQL,con=con_gn,geom_col='geometry')
|
||||
|
||||
|
||||
def get_jdd(module):
|
||||
url = 'https://geonature.cen-isere.fr/geonature/api/monitorings/util/init_data/{mod}'
|
||||
r = requests.get(url.format(mod=module), auth=('user', 'pass'))
|
||||
if r.status_code == 200:
|
||||
d = r.json()
|
||||
return gpd.pd.DataFrame(d['dataset'])[['id_dataset','dataset_name']]
|
||||
|
||||
|
||||
def get_taxon(id_list=100,args={}):
|
||||
url = 'https://geonature.cen-isere.fr/taxhub/api/taxref/allnamebylist/{list}?limit=100000&offset=0'
|
||||
if args is not None:
|
||||
formArgs = '&'.join(['='.join([x,args[x]]) for x in args])
|
||||
url = 'https://geonature.cen-isere.fr/taxhub/api/taxref/?'+formArgs
|
||||
r = requests.get(url.format(list=id_list), auth=('user', 'pass'))
|
||||
if r.status_code == 200:
|
||||
d = r.json()
|
||||
if args is not None:
|
||||
d = r.json()['items']
|
||||
df = gpd.pd.DataFrame(d)
|
||||
return df[['cd_nom','lb_nom','nom_vern']].rename(columns={'lb_nom':'nom_complet'})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
module_name = "chiro"
|
||||
id_tax_list = 100 # taxons occtax
|
||||
|
||||
PATH = '/media/cgeier/SRV/homer_geonat/home/geonatureadmin/protocoles_suivi/{mod}/odk_form/'.format(mod=module_name)
|
||||
PATH = '/home/colas/Documents/9_PROJETS/6_GEONATURE/MODULES/ODK/test/CHIRO/'
|
||||
csv_site_name = 'gn_sites.csv'
|
||||
csv_jdd_name = 'gn_jdds.csv'
|
||||
csv_tax_name = 'gn_taxons.csv'
|
||||
get_site(module_name).to_csv(PATH+csv_site_name,sep=';',index=False)
|
||||
get_jdd(module_name).to_csv(PATH+csv_jdd_name,sep=';',index=False)
|
||||
get_taxon(args={'ordre':'Chiroptera'}).to_csv(PATH+csv_tax_name,sep=';',index=False)
|
||||
|
||||
code_liste = '2'
|
||||
csv_obs_name = 'gn_observateurs.csv'
|
||||
get_list_observateur(code_liste).to_csv(PATH+csv_obs_name,sep=';',index=False)
|
||||
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
|
||||
# SHELL COMMAND
|
||||
# source venv/bin/activate
|
||||
# flask shell
|
||||
|
||||
import os
|
||||
import logging
|
||||
from zipfile import ZipFile
|
||||
import importlib.resources
|
||||
|
||||
from apptax.taxonomie.commands.utils import truncate_bdc_statuts
|
||||
from apptax.database import db
|
||||
from apptax.taxonomie.commands.utils import (
|
||||
copy_from_csv,
|
||||
refresh_taxref_vm,
|
||||
populate_bdc_statut_cor_text_area,
|
||||
)
|
||||
|
||||
logger = logging.getLogger()
|
||||
|
||||
BASE_URL = "/home/geonatureadmin/tmp"
|
||||
zipfile = "BDC-Statuts-v17.zip"
|
||||
status_types_file = "BDC-Statuts-v17/BDC_STATUTS_TYPES_17.csv"
|
||||
status_file = "BDC-Statuts-v17/BDC_STATUTS_17.csv"
|
||||
|
||||
truncate_bdc_statuts()
|
||||
db.session.commit()
|
||||
|
||||
|
||||
archive = ZipFile(os.path.join(BASE_URL, zipfile), "r")
|
||||
with archive.open(status_file) as f:
|
||||
logger.info("Insert BDC statuts…")
|
||||
copy_from_csv(
|
||||
f,
|
||||
"bdc_statut",
|
||||
dest_cols=(
|
||||
"cd_nom",
|
||||
"cd_ref",
|
||||
"cd_sup",
|
||||
"cd_type_statut",
|
||||
"lb_type_statut",
|
||||
"regroupement_type",
|
||||
"code_statut",
|
||||
"label_statut",
|
||||
"rq_statut",
|
||||
"cd_sig",
|
||||
"cd_doc",
|
||||
"lb_nom",
|
||||
"lb_auteur",
|
||||
"nom_complet_html",
|
||||
"nom_valide_html",
|
||||
"regne",
|
||||
"phylum",
|
||||
"classe",
|
||||
"ordre",
|
||||
"famille",
|
||||
"group1_inpn",
|
||||
"group2_inpn",
|
||||
"lb_adm_tr",
|
||||
"niveau_admin",
|
||||
"cd_iso3166_1",
|
||||
"cd_iso3166_2",
|
||||
"full_citation",
|
||||
"doc_url",
|
||||
"thematique",
|
||||
"type_value",
|
||||
),
|
||||
)
|
||||
|
||||
logger.info("Populate BDC statuts…")
|
||||
db.session.execute(
|
||||
importlib.resources.read_text("apptax.migrations.data", "taxonomie_bdc_statuts.sql")
|
||||
)
|
||||
|
||||
populate_bdc_statut_cor_text_area(logger)
|
||||
refresh_taxref_vm()
|
||||
db.session.commit()
|
||||
|
||||
exit()
|
||||
|
||||
# flask taxref link-bdc-statut-to-areas
|
||||
# flask taxref enable-bdc-statut-text --clean -d 38
|
||||
@@ -0,0 +1,160 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
from sqlalchemy import text
|
||||
|
||||
def check_missing_data(con):
|
||||
sql = '''
|
||||
WITH t1 AS (SELECT
|
||||
--tro.*,too.*
|
||||
tro.unique_id_sinp_grp
|
||||
FROM pr_occtax.t_occurrences_occtax AS too
|
||||
JOIN pr_occtax.t_releves_occtax tro USING (id_releve_occtax)
|
||||
WHERE too.id_occurrence_occtax NOT IN (
|
||||
SELECT cco.id_occurrence_occtax FROM pr_occtax.cor_counting_occtax cco
|
||||
)
|
||||
)
|
||||
SELECT
|
||||
s.id_synthese,
|
||||
too.id_releve_occtax,
|
||||
too.id_occurrence_occtax,
|
||||
tro.id_digitiser,
|
||||
s.date_min,
|
||||
dataset_name,
|
||||
dataset_shortname,
|
||||
geom_local geom,
|
||||
meta_device_entry,
|
||||
tro.id_digitiser,
|
||||
tro.date_min,
|
||||
tro.date_max,
|
||||
too.cd_nom,
|
||||
too.nom_cite,
|
||||
s.nom_cite nom_cite_synthese,
|
||||
s.cd_nom cd_nom_synthese,
|
||||
s.last_action
|
||||
FROM pr_occtax.t_occurrences_occtax AS too
|
||||
JOIN pr_occtax.t_releves_occtax tro USING (id_releve_occtax)
|
||||
JOIN gn_meta.t_datasets USING (id_dataset)
|
||||
LEFT JOIN pr_occtax.cor_counting_occtax cco USING (id_occurrence_occtax)
|
||||
LEFT JOIN gn_synthese.synthese s on s.unique_id_sinp = cco.unique_id_sinp_occtax
|
||||
WHERE tro.unique_id_sinp_grp IN (SELECT t1.unique_id_sinp_grp FROM t1)
|
||||
AND (too.nom_cite != s.nom_cite OR s.nom_cite is null)
|
||||
ORDER BY too.id_releve_occtax
|
||||
;
|
||||
'''
|
||||
with con.begin() as cnx:
|
||||
return cnx.execute(text(sql)).fetchall()
|
||||
|
||||
def check_error_synthese(con):
|
||||
sql = '''
|
||||
WITH t1 AS (SELECT
|
||||
--tro.*,too.*
|
||||
tro.unique_id_sinp_grp
|
||||
FROM pr_occtax.t_occurrences_occtax AS too
|
||||
JOIN pr_occtax.t_releves_occtax tro USING (id_releve_occtax)
|
||||
)
|
||||
SELECT
|
||||
s.id_synthese,
|
||||
too.id_releve_occtax,
|
||||
too.id_occurrence_occtax,
|
||||
tro.id_digitiser,
|
||||
s.date_min,
|
||||
dataset_name,
|
||||
dataset_shortname,
|
||||
geom_local geom,
|
||||
meta_device_entry,
|
||||
tro.id_digitiser,
|
||||
tro.date_min,
|
||||
tro.date_max,
|
||||
too.cd_nom,
|
||||
too.nom_cite,
|
||||
s.nom_cite nom_cite_synthese,
|
||||
s.cd_nom cd_nom_synthese,
|
||||
s.last_action
|
||||
FROM pr_occtax.t_occurrences_occtax AS too
|
||||
JOIN pr_occtax.t_releves_occtax tro USING (id_releve_occtax)
|
||||
JOIN gn_meta.t_datasets USING (id_dataset)
|
||||
LEFT JOIN pr_occtax.cor_counting_occtax cco USING (id_occurrence_occtax)
|
||||
LEFT JOIN gn_synthese.synthese s on s.unique_id_sinp = cco.unique_id_sinp_occtax
|
||||
WHERE tro.unique_id_sinp_grp IN (SELECT t1.unique_id_sinp_grp FROM t1)
|
||||
AND (too.nom_cite != s.nom_cite OR s.nom_cite is null)
|
||||
ORDER BY too.id_releve_occtax
|
||||
;
|
||||
'''
|
||||
with con.begin() as cnx:
|
||||
return cnx.execute(text(sql)).fetchall()
|
||||
|
||||
|
||||
def resolve_missing_data(con):
|
||||
sql = '''
|
||||
INSERT INTO pr_occtax.cor_counting_occtax (
|
||||
id_occurrence_occtax,id_nomenclature_life_stage,id_nomenclature_sex,
|
||||
id_nomenclature_obj_count,id_nomenclature_type_count,count_min,count_max,
|
||||
additional_fields)
|
||||
SELECT
|
||||
-- t.operation_type,
|
||||
-- t.operation_date,
|
||||
-- CAST(t.table_content->>'id_counting_occtax' AS BIGINT) id_occurrence_occtax,
|
||||
CAST(t.table_content->>'id_occurrence_occtax' AS BIGINT) AS id_occurrence_occtax,
|
||||
CAST(t.table_content->>'id_nomenclature_life_stage' AS INT) AS id_nomenclature_life_stage,
|
||||
CAST(t.table_content->>'id_nomenclature_sex' AS INT) AS id_nomenclature_sex,
|
||||
CAST(t.table_content->>'id_nomenclature_obj_count' AS INT) AS id_nomenclature_obj_count,
|
||||
CAST(t.table_content->>'id_nomenclature_type_count' AS INT) AS id_nomenclature_type_count,
|
||||
CAST(t.table_content->>'count_min' AS INT) AS count_min,
|
||||
CAST(t.table_content->>'count_max' AS INT) AS count_max,
|
||||
CAST(t.table_content->>'additional_fields' AS JSON) AS additional_fields
|
||||
FROM (
|
||||
SELECT DISTINCT ON (d1.uuid_attached_row)
|
||||
d1.uuid_attached_row, d1.*
|
||||
FROM pr_occtax.t_occurrences_occtax o
|
||||
INNER JOIN gn_commons.t_history_actions o1 ON o.unique_id_occurence_occtax = o1.uuid_attached_row
|
||||
INNER JOIN gn_commons.t_history_actions d1 ON d1.table_content->>'id_occurrence_occtax' = o1.table_content->>'id_occurrence_occtax'
|
||||
INNER JOIN gn_commons.t_history_actions d2 ON d2.uuid_attached_row = d1.uuid_attached_row
|
||||
INNER JOIN gn_commons.t_history_actions o2 ON d2.table_content->>'id_occurrence_occtax' = o2.table_content->>'id_occurrence_occtax'
|
||||
WHERE
|
||||
NOT EXISTS (SELECT NULL FROM pr_occtax.cor_counting_occtax c WHERE o.id_occurrence_occtax = c.id_occurrence_occtax)
|
||||
AND o1.id_table_location = 6 -- ID table t_occurrence_occtax
|
||||
AND d1.id_table_location = 5 -- ID table cor_counting_occtax
|
||||
AND d2.id_table_location = 5 -- ID table cor_counting_occtax
|
||||
AND o2.id_table_location = 6 AND o2.operation_type = 'I' -- table t_occurrence_occtax
|
||||
AND d2.table_content->>'id_occurrence_occtax' <> d1.table_content->>'id_occurrence_occtax'
|
||||
AND d2.operation_date <> d1.operation_date
|
||||
-- AND o1.table_content->>'id_releve_occtax' = '3522'
|
||||
ORDER BY d1.uuid_attached_row, d1.operation_date DESC
|
||||
) AS t
|
||||
;'''
|
||||
with con.begin() as cnx:
|
||||
cnx.execute(text(sql))
|
||||
|
||||
def resolve_synthese_errors(con):
|
||||
sql = '''
|
||||
with t1 as (
|
||||
SELECT
|
||||
--tro.*,too.*
|
||||
tro.unique_id_sinp_grp
|
||||
FROM pr_occtax.t_occurrences_occtax AS too
|
||||
JOIN pr_occtax.t_releves_occtax tro using (id_releve_occtax)
|
||||
),t2 as (
|
||||
select
|
||||
s.id_synthese,
|
||||
too.id_releve_occtax,
|
||||
too.id_occurrence_occtax,
|
||||
too.id_nomenclature_exist_proof
|
||||
FROM pr_occtax.t_occurrences_occtax AS too
|
||||
JOIN pr_occtax.t_releves_occtax tro using (id_releve_occtax)
|
||||
JOIN pr_occtax.cor_counting_occtax cco using (id_occurrence_occtax)
|
||||
JOIN gn_synthese.synthese s on s.unique_id_sinp = cco.unique_id_sinp_occtax
|
||||
WHERE tro.unique_id_sinp_grp IN (SELECT t1.unique_id_sinp_grp FROM t1)
|
||||
AND (too.nom_cite != s.nom_cite)
|
||||
)
|
||||
UPDATE pr_occtax.t_occurrences_occtax too
|
||||
SET id_nomenclature_exist_proof = t2.id_nomenclature_exist_proof
|
||||
FROM t2
|
||||
WHERE too.id_occurrence_occtax = t2.id_occurrence_occtax;
|
||||
'''
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
from pycen import con_gn
|
||||
|
||||
check_missing_data(con_gn)
|
||||
check_error_synthese(con_gn)
|
||||
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
|
||||
import geopandas as gpd
|
||||
import pandas as pd
|
||||
from sqlalchemy.engine import URL
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
# Parametres bdd HOMER - sicen2
|
||||
from pycen import con_sicen
|
||||
schema_hom = 'saisie'
|
||||
table_hom = 'saisie_observation'
|
||||
|
||||
# Parametres bdd HOMER - geonature
|
||||
from pycen import con_gn as con_geo
|
||||
schema_geo = 'taxonomie'
|
||||
table_geo = 'taxref'
|
||||
|
||||
|
||||
file = True
|
||||
path_file = '/home/colas/Documents/9_PROJETS/6_GEONATURE/MODULES/Macrophytes/Module_Macrophytes_Liste_Espèces.xlsx'
|
||||
|
||||
taxref = False
|
||||
args = {
|
||||
'sous_famille': 'Zygaeninae'
|
||||
}
|
||||
|
||||
|
||||
if file:
|
||||
if path_file.split(r'.',1)[-1] == 'xlsx':
|
||||
f = pd.read_excel(path_file)
|
||||
f.columns = f.colufmns.str.lower()
|
||||
lst_cdnom = f.cd_nom.astype(str).unique()
|
||||
elif path_file.rsplit(r'.',1)[-1] == 'csv':
|
||||
f = pd.read_csv(path_file)
|
||||
f.columns = f.columns.str.lower()
|
||||
lst_cdnom = f.cd_nom.astype(str).unique()
|
||||
elif taxref:
|
||||
sql = 'SELECT * FROM {sch}.{tab}'.format(sch = schema_geo,tab = table_geo)
|
||||
sql += ' WHERE %s'%' AND '.join([
|
||||
x+'='+"'%s'"%args[x] if isinstance(args[x],str) else x+' in %s'%str(tuple(args[x])) for x in args
|
||||
])
|
||||
df_tax = pd.read_sql_query(sql,con_geo)
|
||||
lst_cdnom = df_tax.cd_nom.astype(str).unique()
|
||||
else:
|
||||
sql = 'SELECT * FROM {sch}.{tab}'.format(sch=schema_hom, tab=table_hom)
|
||||
gdf_saisie = gpd.read_postgis(
|
||||
sql = sql,
|
||||
con = con_sicen,
|
||||
geom_col = 'geometrie'
|
||||
)
|
||||
lst_cdnom = gdf_saisie.cd_nom.unique()
|
||||
|
||||
# Récupération de la liste taxref non présents
|
||||
# dans la table bib_noms
|
||||
sql = """
|
||||
SELECT DISTINCT cd_nom,cd_ref,nom_vern,id_rang FROM {sch}.{tab}
|
||||
WHERE cd_nom IN ('{lst}')
|
||||
AND cd_nom NOT IN (SELECT cd_nom FROM {sch}.bib_noms WHERE cd_nom IN ('{lst}'))
|
||||
""".format(
|
||||
sch = schema_geo,
|
||||
tab = table_geo,
|
||||
lst = "','".join(lst_cdnom)
|
||||
)
|
||||
df_taxref = pd.read_sql_query(sql,con_geo)
|
||||
df_taxref.rename(columns={'nom_vern':'nom_francais'}, inplace=True)
|
||||
|
||||
# Envoie de la liste dans la table taxonomie.bib_noms
|
||||
df_taxref.drop(columns=['id_rang']).to_sql(
|
||||
name = 'bib_noms',
|
||||
con = con_geo,
|
||||
schema=schema_geo,
|
||||
if_exists='append',
|
||||
index=False
|
||||
)
|
||||
|
||||
# récupération des IDs la table taxonomie.bib_noms
|
||||
sql = 'SELECT distinct cd_nom FROM {sch}.{tab} WHERE cd_nom IN ({id_nom_max})'.format(
|
||||
sch = schema_geo,
|
||||
tab = 'bib_noms',
|
||||
id_nom_max = ",".join(lst_cdnom))
|
||||
|
||||
id_liste = 112 # id_liste de la liste présente dans la table taxonomie.bib_listes
|
||||
sql = """
|
||||
SELECT id_nom FROM {sch}.{tab} WHERE cd_nom IN ({id_nom_max})
|
||||
AND id_nom NOT IN (SELECT id_nom FROM {sch}.cor_nom_liste WHERE id_liste = '{id_liste}')
|
||||
""".format(
|
||||
sch = schema_geo,
|
||||
tab = 'bib_noms',
|
||||
id_nom_max = ",".join(lst_cdnom),
|
||||
id_liste = id_liste
|
||||
)
|
||||
_df = pd.read_sql_query(sql,con_geo)
|
||||
# _df = pd.read_sql_table(
|
||||
# table_name='bib_noms',
|
||||
# con= con_geo,
|
||||
# schema=schema_geo
|
||||
# )
|
||||
_df['id_liste'] = id_liste # id_liste de la liste présente dans la table taxonomie.bib_listes
|
||||
df = _df[['id_liste','id_nom']]
|
||||
# envoie de la liste dans la table taxonomie.bib_noms
|
||||
df.to_sql(
|
||||
name = 'cor_nom_liste',
|
||||
con = con_geo,
|
||||
schema=schema_geo,
|
||||
if_exists='append',
|
||||
index=False
|
||||
)
|
||||
|
||||
sql = 'SELECT * FROM {sch}.cor_nom_liste'.format(
|
||||
sch = schema_geo)
|
||||
cor_nom_liste = pd.read_sql_query(sql,con_geo)
|
||||
@@ -0,0 +1,29 @@
|
||||
|
||||
import pandas as pd
|
||||
import geopandas as gpd
|
||||
|
||||
from pycen import con_gn
|
||||
|
||||
path = '/home/colas/Documents/5_BDD/TAXON/TAXREF/TAXREF_V14_2020/'
|
||||
cdnom_disparu = pd.read_excel(path+'CDNOM_DISPARUS.xls')
|
||||
cdnom_disparu.columns = cdnom_disparu.columns.str.lower()
|
||||
|
||||
def get_bib_list(name):
|
||||
sql = """SELECT bn.*
|
||||
FROM taxonomie.bib_noms bn
|
||||
JOIN taxonomie.cor_nom_liste c USING (id_nom)
|
||||
JOIN taxonomie.bib_listes bl USING (id_liste)
|
||||
WHERE lower(bl.nom_liste) = '%s';
|
||||
"""%name.lower()
|
||||
return pd.read_sql_query(sql,con_gn)
|
||||
|
||||
amp = get_bib_list('Amphibiens')
|
||||
amp[amp.cd_nom.isin(cdnom_disparu.cd_nom_remplacement)]
|
||||
odo = get_bib_list('Odonates')
|
||||
odo[odo.cd_nom.isin(cdnom_disparu.cd_nom_remplacement)]
|
||||
flo = get_bib_list('Flore')
|
||||
flo[flo.cd_nom.isin(cdnom_disparu.cd_nom_remplacement)]
|
||||
|
||||
cdnom_disparu[cdnom_disparu.cd_nom_remplacement.isin(amp.cd_nom)]
|
||||
cdnom_disparu[cdnom_disparu.cd_nom_remplacement.isin(flo.cd_nom)]
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
|
||||
|
||||
from pycen import con_gn as con
|
||||
import pandas as pd
|
||||
|
||||
|
||||
sch = 'ref_nomenclatures'
|
||||
tab = ['t_nomenclatures','bib_nomenclatures_types']
|
||||
sql = """
|
||||
SELECT
|
||||
a.id_nomenclature,
|
||||
--a.cd_nomenclature,
|
||||
a.mnemonique mnemo,
|
||||
a.label_default as label,
|
||||
a.definition_default def,
|
||||
b.mnemonique bib_mnemo,
|
||||
b.label_default bib_label,
|
||||
b.definition_default bib_def,
|
||||
a.active
|
||||
FROM {sch}.{tab0} a
|
||||
JOIN {sch}.{tab1} b USING (id_type)
|
||||
""".format(sch=sch,tab0=tab[0],tab1=tab[1])
|
||||
df = pd.read_sql_query(sql,con)
|
||||
df.set_index('id_nomenclature', inplace=True)
|
||||
|
||||
|
||||
info_releve = [
|
||||
"Type de regroupement", # Type de relevé/regroupement
|
||||
"Type d'information géographique",
|
||||
"Nature d'objet géographique",
|
||||
"Méthode de regroupement",
|
||||
]
|
||||
infos_occ_sens = [
|
||||
# "Techniques d'observation", # "Techniques d'observation"
|
||||
"Statut biologique",
|
||||
"Etat biologique de l'observation", # Etat biologique
|
||||
"Statut biogéographique",
|
||||
"Niveau de naturalité", # Naturalité
|
||||
"Niveaux de sensibilité", # Sensibilité
|
||||
"Niveaux de précision de diffusion souhaités", # Niveau de diffusion
|
||||
"Statut d'observation",
|
||||
"Existence d'un floutage sur la donnée", # Niveau de Floutage
|
||||
"Statut de la source",
|
||||
# "Référence bibliographique",
|
||||
"Comportement des occurrences observées", # Comportement
|
||||
]
|
||||
|
||||
infos_denombremt = [
|
||||
"Stade de vie : stade de développement du sujet", # Stade de vie
|
||||
"Sexe",
|
||||
"Type de dénombrement", # Type du dénombrement
|
||||
"Objet du dénombrement",
|
||||
]
|
||||
infos_determ_valid = [
|
||||
"Méthode de détermination",
|
||||
# "Existence d'une preuve",
|
||||
# "Preuve numérique",
|
||||
# "Preuve non-numérique",
|
||||
"Statut de validation",
|
||||
]
|
||||
df[df.bib_label.isin(info_releve)]
|
||||
df[df.bib_label.isin(infos_occ_sens)]
|
||||
df[df.bib_label.isin(infos_denombremt)]
|
||||
df[df.bib_label.isin(infos_determ_valid)]
|
||||
|
||||
|
||||
for i in df.sort_values('bib_label').bib_label.unique():
|
||||
if 'Télédétection' not in df[df.bib_label==i].label.tolist():
|
||||
continue
|
||||
print()
|
||||
print(i+' :')
|
||||
print(df[df.bib_label==i].label.tolist())
|
||||
|
||||
|
||||
lst_champs = [
|
||||
"Type de regroupement",
|
||||
# "Type d'information géographique",
|
||||
# "Nature d'objet géographique",
|
||||
# "Méthodes d'observation",
|
||||
# "Statut biologique",
|
||||
# "Etat biologique de l'observation",
|
||||
# "Statut biogéographique",
|
||||
# "Niveau de naturalité",
|
||||
# "Niveaux de sensibilité",
|
||||
# "Niveaux de précision de diffusion souhaités",
|
||||
# "Statut d'observation",
|
||||
# "Existence d'un floutage sur la donnée",
|
||||
# "Statut de la source",
|
||||
# "Comportement des occurrences observées",
|
||||
# "Stade de vie : stade de développement du sujet",
|
||||
# "Sexe",
|
||||
# "Type de dénombrement",
|
||||
# "Objet du dénombrement",
|
||||
# "Méthode de détermination",
|
||||
# "Statut de validation",
|
||||
]
|
||||
df[df.bib_label.isin(lst_champs)].bib_def.unique()
|
||||
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
|
||||
|
||||
import geopandas as gpd
|
||||
from geoalchemy2 import Geometry
|
||||
import pycen
|
||||
import pycen.wfs
|
||||
|
||||
# con = conn
|
||||
con = pycen.con_gn
|
||||
|
||||
def update_larea(df, cols_updt=[]):
|
||||
from pycen import update_to_sql
|
||||
table = 'l_areas'
|
||||
idtyp = tuple(df.id_type.unique())
|
||||
if len(idtyp) > 1:
|
||||
where = 'in %s'%tuple(df.id_type.unique())
|
||||
else :
|
||||
where = '=%s'%tuple(df.id_type.unique())
|
||||
sql = 'SELECT id_area, area_name FROM ref_geo.l_areas WHERE id_type %s;'%where
|
||||
larea = gpd.pd.read_sql_query(sql,con)
|
||||
|
||||
to_updt = df.merge(larea, on='area_name')
|
||||
update_to_sql(
|
||||
df=to_updt[['id_area',*cols_updt]],
|
||||
con=con,
|
||||
table_name=table,
|
||||
schema_name='ref_geo',
|
||||
key_name='id_area',
|
||||
)
|
||||
|
||||
|
||||
dic = {
|
||||
'ID_ENS':'area_code',
|
||||
'ENS_Id':'area_code',
|
||||
'Id':'area_code',
|
||||
'ID':'area_code',
|
||||
'NOM':'area_name',
|
||||
'Nom_ens':'area_name',
|
||||
# 'URL':'source',
|
||||
}
|
||||
# pycen.wfs.list_layer('http://ws.carmencarto.fr/WFS/94/ENS_CG38')
|
||||
df = pycen.wfs.get_wfs('http://ws.carmencarto.fr/WFS/94/ENS_CG38', 'zone_d_intervention_02')
|
||||
df.rename(columns=dic,inplace=True)
|
||||
df = df[[*set(dic.values()),'geometry']]
|
||||
df.rename_geometry('geom', inplace=True)
|
||||
df['id_type'] = 35
|
||||
df['geojson_4326'] = df.to_crs(4326).geom.__geo_interface__['features']
|
||||
df['geojson_4326'] = [x['geometry'] for x in df['geojson_4326']]
|
||||
# df['geojson_4326'] = df['geojson_4326'].astype(str)
|
||||
df['centroid'] = df.geom.centroid.to_wkt()
|
||||
df['enable'] = True
|
||||
df['source'] = 'http://ws.carmencarto.fr/WFS/94/ENS_CG38'
|
||||
|
||||
df.to_postgis(
|
||||
name='l_areas',
|
||||
con=con,
|
||||
schema='ref_geo',
|
||||
if_exists='append',
|
||||
index=False,
|
||||
index_label=None,
|
||||
chunksize=None,
|
||||
dtype={
|
||||
'centroid': Geometry(geometry_type='POINT',srid=2154)
|
||||
},
|
||||
)
|
||||
|
||||
# path0 = '/home/colas/Documents/5_BDD/BASSIN_VERSANT/'
|
||||
path0 = '/home/colas/Documents/9_PROJETS/6_GEONATURE/AREA/'
|
||||
dic = {
|
||||
# 'ID_MNHN':'area_code',
|
||||
'NOM':'area_name',
|
||||
'URL':'source',
|
||||
}
|
||||
# path = 'BD_Topage_FXX_2022-shp/BD_TOPAGE_2022-shp/BassinVersantTopographique_FXX-shp/'
|
||||
# file = 'BassinVersantTopographique_FXX.shp'
|
||||
path = 'IN DATA BASE/znieff1/'
|
||||
file = 'znieff1.shp'
|
||||
id_type = 3 # id_type in bdd geonature : ref_geo.bib_areas_types
|
||||
# 3 : 'ZNIEFF1',
|
||||
# 4 : 'APB',
|
||||
# 5 : 'RNN',
|
||||
# 6 : 'RNR',
|
||||
# 7 : 'ZPS',
|
||||
# 8 : 'SIC',
|
||||
# 9 : 'ZICO',
|
||||
# 10 : 'RNCFS',
|
||||
# 12 : 'SCEN',
|
||||
# 15 : 'PNR',
|
||||
# 16 : 'RBIOL',
|
||||
# 20 : 'AA',
|
||||
# 32 : 'Zones biogéographiques',
|
||||
|
||||
# DIC for pr_zh
|
||||
# dic = {
|
||||
# # 'index' : 'id_hydro',
|
||||
# 'index' : 'id_rb',
|
||||
# 'TopoOH' : 'name',
|
||||
# 'lib_ssbv' : 'name',
|
||||
# }
|
||||
|
||||
# sql = '''
|
||||
# SELECT a.* FROM ref_geo.l_areas a
|
||||
# WHERE a.id_type = 25
|
||||
# '''
|
||||
sql = '''
|
||||
SELECT a.* FROM ref_geo.l_areas a
|
||||
JOIN ref_geo.bib_areas_types b USING (id_type)
|
||||
WHERE b.type_code = 'DEP' AND a.area_code = '38'
|
||||
'''
|
||||
reg = gpd.read_postgis(sql,con)
|
||||
|
||||
|
||||
df = gpd.read_file(path0+path+file)
|
||||
if file == 'N_ENP_PN_S_000.shp':
|
||||
if id_type == 20:
|
||||
df = df[df.CODE_R_ENP == 'AAPN']
|
||||
elif id_type == 1:
|
||||
df = df[df.CODE_R_ENP == 'CPN']
|
||||
else:
|
||||
raise Exception('ERROR N_ENP_PN_S_000.shp')
|
||||
if file == 'N_ENP_SCEN_S_000.shp':
|
||||
df = df[df.ID_LOCAL.str.contains('CENIS')]
|
||||
if file != 'N_ENP_SCEN_S_000.shp':
|
||||
intersect = gpd.sjoin(df,reg,op='intersects').index
|
||||
df = df[df.index.isin(intersect)]
|
||||
|
||||
|
||||
# SPEC t_river_basin
|
||||
# df.reset_index(drop=True,inplace=True)
|
||||
# df.reset_index(drop=False,inplace=True)
|
||||
# df.loc[df['index']==0,'index'] = 28
|
||||
# df.loc[df['index']==1,'index'] = 29
|
||||
# df['lib_ssbv'] = df['lib_ssbv']+' - '+df['code_ssbv']
|
||||
|
||||
df.rename_geometry('geom', inplace=True)
|
||||
df.dropna(how='all',axis=1,inplace=True)
|
||||
df = pycen.tools.Polygons_to_MultiPolygon(df)
|
||||
df.rename(columns=dic, inplace=True)
|
||||
# df['geom'] = df.geometry.to_wkt().copy()
|
||||
del_col = df.columns[~df.columns.isin(['geom',*[*dic.values()]])]
|
||||
del_col = df.columns[~df.columns.isin([*[*dic.values()]])]
|
||||
df.drop(columns=del_col, inplace=True)
|
||||
df['id_type'] = id_type
|
||||
|
||||
update_larea(df,cols_updt=['area_code'])
|
||||
|
||||
# SPEC t_river_basin / t_hydro_area
|
||||
# df.to_crs(4326,inplace=True)
|
||||
# df.geom = df.geom.to_wkt().copy()
|
||||
# df.to_wkt().to_sql(
|
||||
# name='t_river_basin', con=con, schema='pr_zh',
|
||||
# if_exists='append', index=False,
|
||||
# dtype={
|
||||
# 'geom': Geometry(srid=4326)
|
||||
# # 'id_lim_list':uuid.SafeUUID
|
||||
# }
|
||||
# )
|
||||
|
||||
df['geojson_4326'] = df.to_crs(4326).geom.__geo_interface__['features']
|
||||
df['geojson_4326'] = [x['geometry'] for x in df['geojson_4326']]
|
||||
# df['geojson_4326'] = df['geojson_4326'].astype(str)
|
||||
df['centroid'] = df.geom.centroid.to_wkt()
|
||||
df['enable'] = True
|
||||
df['source'] = 'IGN'
|
||||
|
||||
df.to_postgis(
|
||||
name='l_areas',
|
||||
con=con,
|
||||
schema='ref_geo',
|
||||
if_exists='append',
|
||||
index=False,
|
||||
index_label=None,
|
||||
chunksize=None,
|
||||
dtype={
|
||||
'centroid': Geometry(geometry_type='POINT',srid=2154)
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
sql = '''
|
||||
SELECT a.id_area,a.area_name,a.geom FROM ref_geo.l_areas a
|
||||
JOIN ref_geo.bib_areas_types b USING (id_type)
|
||||
WHERE b.type_code = 'COM'
|
||||
'''
|
||||
tu = gpd.read_postgis(sql,con_gn)
|
||||
|
||||
sql = '''
|
||||
SELECT a.id_zh,a.code,
|
||||
ST_MakeValid(ST_Transform(ST_SetSRID(a.geom,4326),2154)) geom
|
||||
FROM pr_zh.t_zh a
|
||||
--JOIN ref_geo.bib_areas_types b USING (id_type)
|
||||
--WHERE b.type_code = 'COM'
|
||||
'''
|
||||
to = gpd.read_postgis(sql,con_gn)
|
||||
@@ -0,0 +1,311 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
|
||||
def test_data(con,tab,col,status):
|
||||
sql = '''
|
||||
SELECT count({col}) FROM {sch}.{tab} WHERE {col} = '{status}'
|
||||
;'''.format(sch='taxonomie',tab=tab,col=col,status=status)
|
||||
with con.begin() as cnx:
|
||||
return cnx.execute(sql).one()[0]
|
||||
|
||||
def test_cor_values(con,vals):
|
||||
sql = '''
|
||||
SELECT count(id_value_text) FROM {sch}.{tab} WHERE (id_value,id_text) = {vals}
|
||||
;'''.format(sch='taxonomie',tab='bdc_statut_cor_text_values',vals=vals)
|
||||
with con.begin() as cnx:
|
||||
return cnx.execute(sql).one()[0]
|
||||
|
||||
def test_status_type(con,col,status):
|
||||
sql = '''
|
||||
SELECT count({col}) FROM {sch}.{tab} WHERE {col} = '{status}'
|
||||
;'''.format(sch='taxonomie',tab='bdc_statut_type',col=col,status=status)
|
||||
with con.begin() as cnx:
|
||||
return cnx.execute(sql).one()[0]
|
||||
|
||||
def insert_status_alerte(con):
|
||||
"""
|
||||
Insert type de statut pour les alertes
|
||||
|
||||
Si la valeur AL n'existe pas dans la table bdc_statut_type, on l'insert
|
||||
"""
|
||||
if test_data(con,'bdc_statut_type','cd_type_statut','AL') > 0:
|
||||
# if test_status_type(con,'cd_type_statut','AL') > 0:
|
||||
print('ALERTE STATUS ALREADY EXISTS')
|
||||
else:
|
||||
sql = '''
|
||||
INSERT INTO {sch}.{tab} (cd_type_statut,lb_type_statut,regroupement_type,thematique,type_value) VALUES
|
||||
('AL','Liste d''alerte départementale','Alerte','STATUTS','VALUE')
|
||||
;'''.format(sch='taxonomie',tab='bdc_statut_type')
|
||||
with con_gn.begin() as cnx:
|
||||
cnx.execute(sql)
|
||||
|
||||
def insert_status_values(con):
|
||||
"""
|
||||
Inserts predefined status values into the 'bdc_statut_values' table if they do not already exist.
|
||||
|
||||
This function iterates over a list of status values, checking whether each value already exists in the
|
||||
specified database table. If a value does not exist, it inserts the value into the table. Status values
|
||||
include a code and a label, which describe the extinction risk or conservation status of a taxonomic
|
||||
group at the departmental level.
|
||||
|
||||
Args:
|
||||
con: A SQLAlchemy connection object to the database.
|
||||
|
||||
Note:
|
||||
This function assumes the existence of a schema named 'taxonomie' and a table named
|
||||
'bdc_statut_values' in the database connected via 'con'. It also assumes the availability of a
|
||||
'con_gn' connection for executing the SQL commands.
|
||||
"""
|
||||
|
||||
vals = [
|
||||
['RE','Disparue au niveau départemental'],
|
||||
['AS-1','Quasi menacée (localisées sans signe de déclin)'],
|
||||
['AS-2','Quasi menacée (répandues mais en déclin)'],
|
||||
['AS-3','Quasi menacée (répandues, déclin à confirmer)']
|
||||
]
|
||||
for val in vals:
|
||||
if test_data(con,'bdc_statut_values','label_statut',val[1]) > 0:
|
||||
print('ALERTE VALUE STATUS ALREADY EXISTS : ',val[1])
|
||||
else:
|
||||
sql = '''
|
||||
INSERT INTO {sch}.{tab} (code_statut,label_statut) VALUES
|
||||
('{val0}','{val1}')
|
||||
'''.format(sch='taxonomie',tab='bdc_statut_values',val0=val[0],val1=val[1])
|
||||
with con_gn.begin() as cnx:
|
||||
cnx.execute(sql)
|
||||
|
||||
def get_text_id(con,cd_doc):
|
||||
sql = '''
|
||||
SELECT id_text FROM {sch}.{tab} WHERE cd_doc = '{cd_doc}'
|
||||
;'''.format(sch='taxonomie',tab='bdc_statut_text',cd_doc=cd_doc)
|
||||
with con.begin() as cnx:
|
||||
return cnx.execute(sql).one()[0]
|
||||
|
||||
def get_area_id(con,area):
|
||||
sql = '''
|
||||
SELECT id_area FROM {sch}.{tab} WHERE area_name = '{area}'
|
||||
;'''.format(sch='ref_geo',tab='l_areas',area=area)
|
||||
with con.begin() as cnx:
|
||||
return cnx.execute(sql).one()[0]
|
||||
|
||||
def get_values_id(con,col,terme):
|
||||
if isinstance(terme,int|str):
|
||||
terme = [terme]
|
||||
else:
|
||||
terme = list(terme)
|
||||
sql = '''
|
||||
SELECT id_value FROM {sch}.{tab} WHERE {col} IN {terme} AND label_statut <> 'Disparue au niveau régional'
|
||||
;'''.format(sch='taxonomie',tab='bdc_statut_values',col=col,terme=tuple(terme)).replace(',)',')')
|
||||
with con.begin() as cnx:
|
||||
return cnx.execute(sql).all()
|
||||
|
||||
def insert_status_cor_text_area(con,id_doc):
|
||||
id_text = get_text_id(con,id_doc)
|
||||
id_area = get_area_id(con,'Isère')
|
||||
|
||||
check_sql = '''
|
||||
SELECT count(id_text) FROM {sch}.{tab} WHERE (id_text,id_area) = ({id_text},{id_area})
|
||||
;'''.format(sch='taxonomie',tab='bdc_statut_cor_text_area',id_text=id_text,id_area=id_area)
|
||||
with con.begin() as cnx:
|
||||
if cnx.execute(check_sql).one()[0] == 0:
|
||||
sql = ('''
|
||||
INSERT INTO {sch}.{tab} (id_text,id_area) VALUES ({id_text},{id_area})
|
||||
;'''.format(sch='taxonomie',tab='bdc_statut_cor_text_area',id_text=id_text,id_area=id_area))
|
||||
cnx.execute(sql)
|
||||
|
||||
|
||||
def insert_status_cor_text_values(con,id_doc,values):
|
||||
insert_status_values(con)
|
||||
id_text = get_text_id(con,id_doc)
|
||||
id_vals = [x[0] for x in get_values_id(con,'code_statut',values)]
|
||||
|
||||
zip_vals = tuple(zip(id_vals,[id_text]*len(id_vals)))
|
||||
cor_vals = [x for x in zip_vals if test_cor_values(con,x)==0]
|
||||
|
||||
sql = ('''
|
||||
INSERT INTO {sch}.{tab} (id_value,id_text) VALUES {values}
|
||||
;'''
|
||||
.format(sch='taxonomie',tab='bdc_statut_cor_text_values',values=cor_vals)
|
||||
.replace(r'[','')
|
||||
.replace(r']',''))
|
||||
|
||||
with con_gn.begin() as cnx:
|
||||
cnx.execute(sql)
|
||||
|
||||
def get_id_status_cor_text_values(con,id_doc,values):
|
||||
id_text = get_text_id(con,id_doc)
|
||||
id_vals = [x[0] for x in get_values_id(con,'code_statut',values)]
|
||||
|
||||
zip_vals = tuple(zip(id_vals,[id_text]*len(id_vals)))
|
||||
cor_vals = tuple(x for x in zip_vals if test_cor_values(con,x)>0)
|
||||
|
||||
sql = ('''
|
||||
SELECT id_value_text FROM {sch}.{tab} WHERE (id_value,id_text) IN {cor_vals}
|
||||
;'''
|
||||
.format(sch='taxonomie',tab='bdc_statut_cor_text_values',cor_vals=cor_vals)
|
||||
.replace('),)','))'))
|
||||
|
||||
with con.begin() as cnx:
|
||||
return cnx.execute(sql).all()
|
||||
|
||||
def test_status_text(con,col,cd_doc):
|
||||
sql = '''
|
||||
SELECT count({col}) FROM {sch}.{tab} WHERE {col} = '{cd_doc}'
|
||||
;'''.format(sch='taxonomie',tab='bdc_statut_text',col=col,cd_doc=cd_doc)
|
||||
with con.begin() as cnx:
|
||||
return cnx.execute(sql).one()[0]
|
||||
|
||||
def insert_statut_text(con,cd_doc,doc,cd_sig,lb_adm_tr,niveau_admin):
|
||||
if test_data(con,'bdc_statut_text','cd_doc',cd_doc['id_doc']) > 0:
|
||||
print('ALERTE TEXT STATUS ALREADY EXISTS : ',doc)
|
||||
else:
|
||||
sql = '''
|
||||
INSERT INTO {sch}.{tab} (cd_type_statut,cd_doc,cd_sig,niveau_admin,lb_adm_tr,doc_url,enable) VALUES
|
||||
('AL',{cd_doc},'{cdsig}','{lb_adm}','{niv_adm}','{doc}',TRUE)
|
||||
;'''.format(
|
||||
sch='taxonomie',
|
||||
tab='bdc_statut_text',
|
||||
cd_doc=cd_doc['id_doc'],
|
||||
doc=doc,
|
||||
cdsig=cd_sig,
|
||||
lb_adm=lb_adm_tr,
|
||||
niv_adm=niveau_admin
|
||||
)
|
||||
with con_gn.begin() as cnx:
|
||||
cnx.execute(sql)
|
||||
|
||||
insert_status_cor_text_area(con,cd_doc['id_doc'])
|
||||
insert_status_cor_text_values(con,cd_doc['id_doc'],cd_doc['id_values'])
|
||||
|
||||
def get_cd_ref(con,cd_nom):
|
||||
sql = '''
|
||||
SELECT cd_ref FROM {sch}.{tab} WHERE cd_nom = '{cd_nom}'
|
||||
;'''.format(sch='taxonomie',tab='taxref',cd_nom=cd_nom)
|
||||
with con.begin() as cnx:
|
||||
return cnx.execute(sql).one()[0]
|
||||
|
||||
def get_max_idstatuttaxo(con):
|
||||
sql = '''
|
||||
SELECT max(id) FROM {sch}.{tab}
|
||||
;'''.format(sch='taxonomie',tab='bdc_statut_taxons')
|
||||
with con.begin() as cnx:
|
||||
return cnx.execute(sql).one()[0]
|
||||
|
||||
def insert_status_taxo(con,cd_nom,cd_doc,status):
|
||||
id_statut_cor = get_id_status_cor_text_values(con,cd_doc,status)[0][0]
|
||||
cd_ref = get_cd_ref(con,cd_nom)
|
||||
id_statut_taxo = get_max_idstatuttaxo(con) + 1
|
||||
|
||||
sql_check = '''
|
||||
SELECT count(id) FROM {sch}.{tab}
|
||||
WHERE id_value_text = '{id_statut_cor}' AND cd_nom = '{cd_nom}' AND cd_ref = '{cd_ref}'
|
||||
;'''.format(sch='taxonomie',tab='bdc_statut_taxons',cd_ref=cd_ref,cd_nom=cd_nom,id_statut_cor=id_statut_cor)
|
||||
with con.begin() as cnx:
|
||||
check = cnx.execute(sql_check).one()[0]
|
||||
|
||||
if check == 0:
|
||||
sql = '''
|
||||
INSERT INTO {sch}.{tab} (id,id_value_text,cd_nom,cd_ref) VALUES
|
||||
('{id_statut_taxo}','{id_statut_cor}','{cd_nom}','{cd_ref}')
|
||||
;'''.format(sch='taxonomie',tab='bdc_statut_taxons',id_statut_taxo=id_statut_taxo,cd_ref=cd_ref,cd_nom=cd_nom,id_statut_cor=id_statut_cor)
|
||||
with con_gn.begin() as cnx:
|
||||
cnx.execute(sql)
|
||||
|
||||
def get_status_type(con,col,status):
|
||||
sql = '''
|
||||
SELECT * FROM {sch}.{tab} WHERE {col} = '{status}'
|
||||
;'''.format(sch='taxonomie',tab='bdc_statut_type',col=col,status=status)
|
||||
return pd.read_sql(sql,con)
|
||||
|
||||
def get_taxonomie(con,cd_nom):
|
||||
if isinstance(cd_nom,int):
|
||||
cd_nom = [cd_nom]
|
||||
else:
|
||||
cd_nom = list(cd_nom)
|
||||
sql = '''
|
||||
SELECT cd_nom,cd_ref,cd_sup,lb_nom,lb_auteur,nom_complet_html,nom_valide,regne,phylum,classe,ordre,famille,group1_inpn,group2_inpn FROM {sch}.{tab} WHERE cd_nom IN {cd_nom}
|
||||
;'''.format(sch='taxonomie',tab='taxref',cd_nom=tuple(cd_nom)).replace(",)",")")
|
||||
return pd.read_sql(sql,con)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
import pandas as pd
|
||||
# Définition de la connection à la bdd GéoNature
|
||||
from pycen import con_gn
|
||||
|
||||
# Le premier feuillet du fichier Excel lu.
|
||||
# Composition minimale: [CD_NOM,Statut, Source, Source_url]
|
||||
# ATTENTION:
|
||||
# - le CD_NOM doit correspondre au CD_NOM de la table taxref
|
||||
# - Les taxons dont la Source_url est None ou NA seront ignorés
|
||||
file = '/home/colas/Documents/9_PROJETS/6_GEONATURE/listes_alertes_isère.xlsx'
|
||||
# Echelle administratif des listes à implémenter
|
||||
niveau_admin = 'Département'
|
||||
# Nom du niveau adminsitratif
|
||||
lb_adm_tr = 'Isère'
|
||||
# Code SIG du niveau adminnistratif
|
||||
cd_sig = 'INSEED38'
|
||||
|
||||
insert_status_alerte(con_gn)
|
||||
# Dictionnaire des listes d’alerte à intégrer,
|
||||
# identifiant et codes status respectivement concernés
|
||||
cd_doc = {
|
||||
'Statut_de_conservation_des_poissons_et_écrevisses_en_Isère_2015':{
|
||||
'id_doc':999990,
|
||||
'id_values':['RE','CR','EN','VU','NT','LC','DD','NA',]
|
||||
},
|
||||
'Liste_d’alerte_sur_les_orthoptères_menacés_en_Isère_2014':{
|
||||
'id_doc':999991,
|
||||
'id_values':['RE','CR','EN','VU','AS-1','AS-2','AS-3','LC','DD','NA']
|
||||
},
|
||||
'Statuts_de_conservation_de_la_faune_sauvage_en_isere_2016':{
|
||||
'id_doc':999992,
|
||||
'id_values':['RE','CR','EN','VU','NT','LC','DD','NA','NE',]
|
||||
},
|
||||
'Liste_rouge_des_Odonates_de_l’Isère_2013':{
|
||||
'id_doc':999993,
|
||||
'id_values':['RE','CR','EN','VU','NT','LC','DD','NA',]
|
||||
},
|
||||
'Liste_rouge_des_lépidoprere_rhopaloceres_et_zygenes_de_l’Isère_2015':{
|
||||
'id_doc':999994,
|
||||
'id_values':['RE','CR','EN','VU','NT','LC','DD','NA','NE','EX']
|
||||
},
|
||||
}
|
||||
|
||||
df = (pd.read_excel(file,keep_default_na=False)
|
||||
.rename(columns={
|
||||
'Statut':'code_statut',
|
||||
'CD_NOM':'cd_nom',
|
||||
'Source':'source',
|
||||
'Source_url':'doc_url',
|
||||
}))
|
||||
df = df.loc[df.doc_url!='']
|
||||
for d in df.source.unique():
|
||||
doc_url = df.loc[df.source==d,'doc_url'].unique()[0]
|
||||
insert_statut_text(con_gn,cd_doc[d],doc_url,cd_sig,lb_adm_tr,niveau_admin)
|
||||
|
||||
# INSERTION dans la table bdc_statut_taxons
|
||||
# Boucle sur chaque taxon. Peu prendre quelques minutes.
|
||||
[
|
||||
insert_status_taxo(
|
||||
con = con_gn,
|
||||
cd_nom = row.cd_nom,
|
||||
cd_doc = cd_doc[row.source]['id_doc'],
|
||||
status = row.code_statut)
|
||||
for row in df.itertuples()
|
||||
]
|
||||
|
||||
st = get_status_type(con_gn,'cd_type_statut','AL')
|
||||
for c in st:
|
||||
df[c] = st[c][0]
|
||||
tax = get_taxonomie(con_gn,df['cd_nom'])
|
||||
del tax['nom_valide']
|
||||
del df['source']
|
||||
del df['nom_français']
|
||||
del df['nom_latin']
|
||||
df = df.merge(tax,how='inner',on='cd_nom')
|
||||
df['cd_sig'] = cd_sig
|
||||
df['lb_adm_tr'] = lb_adm_tr
|
||||
df['niveau_admin'] = niveau_admin
|
||||
df.to_sql('bdc_statut',con_gn,schema='taxonomie',if_exists='append',index=False)
|
||||
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
|
||||
from sqlalchemy import dialects
|
||||
from pycen import con_gn
|
||||
import pandas as pd
|
||||
import json
|
||||
import ast
|
||||
import os
|
||||
|
||||
DIR = '/home/colas/Documents/9_PROJETS/6_GEONATURE'
|
||||
FILE = 'IMPORTS/OBSERVATEURS/Table Observateurs Platière.csv'
|
||||
|
||||
|
||||
def to_organismes(df):
|
||||
sch = 'utilisateurs'
|
||||
tab = 'bib_organismes'
|
||||
org = pd.read_sql_table(tab,con_gn,sch)
|
||||
df = df[~df.nom_organisme.isin(org.nom_organisme)]
|
||||
df.to_sql(
|
||||
tab,con_gn,sch,
|
||||
if_exists='append',
|
||||
index=False)
|
||||
|
||||
|
||||
def get_organismes(LIST):
|
||||
sch = 'utilisateurs'
|
||||
tab = 'bib_organismes'
|
||||
formlist = LIST.replace({"'":"''"},regex=True)
|
||||
formlist = str(tuple(formlist)).replace('"',"'")
|
||||
sql = '''
|
||||
SELECT id_organisme,nom_organisme FROM {sch}.{tab}
|
||||
WHERE nom_organisme IN {n_org}
|
||||
;'''.format(sch=sch,tab=tab,n_org=formlist)
|
||||
|
||||
return pd.read_sql_query(sql, con_gn)
|
||||
|
||||
|
||||
def to_roles(df):
|
||||
sch = 'utilisateurs'
|
||||
tab = 't_roles'
|
||||
dtypes = {}
|
||||
if 'champs_addi' in df.columns:
|
||||
dtypes = {**dtypes,'champs_addi':dialects.postgresql.JSONB}
|
||||
|
||||
df.to_sql(
|
||||
tab,con_gn,sch,
|
||||
if_exists='append',
|
||||
index=False,
|
||||
dtype=dtypes)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
df = pd.read_csv(os.path.join(DIR,FILE))
|
||||
df = df[df.rqe!='à ne pas importer']
|
||||
obs = df[['organisme_long']]\
|
||||
.drop_duplicates()\
|
||||
.rename(columns={'organisme_long':'nom_organisme'})
|
||||
to_organismes(obs)
|
||||
|
||||
org = get_organismes(obs.nom_organisme)
|
||||
dic_org = dict(zip(org.nom_organisme,org.id_organisme))
|
||||
df.organisme_long.replace(dic_org,inplace=True)
|
||||
|
||||
role = df[['nom','prenom','organisme_long','statut','poste']]\
|
||||
.rename(columns={
|
||||
'nom':'nom_role',
|
||||
'prenom':'prenom_role',
|
||||
'organisme_long':'id_organisme',
|
||||
})
|
||||
role['champs_addi'] = json.loads(
|
||||
role[['statut','poste']].to_json(orient="records",force_ascii=False)
|
||||
)
|
||||
role.drop(columns=['statut','poste'],inplace=True)
|
||||
to_roles(role)
|
||||
@@ -0,0 +1,44 @@
|
||||
from pycen import con_gn
|
||||
from sqlalchemy.sql import text
|
||||
|
||||
def update_additional_values(
|
||||
con,table='cor_counting_occtax',shema='pr_occtax',
|
||||
jsonb_column='additional_fields',keyfield='code_atlas'):
|
||||
sql = text('''
|
||||
UPDATE %(sch)s.%(tab)s
|
||||
SET "%(jsonb_c)s" = JSONB_SET(
|
||||
"%(jsonb_c)s",
|
||||
'{%(keyf)s}'::text[],
|
||||
TO_JSONB(
|
||||
ref_nomenclatures.get_nomenclature_label(("%(jsonb_c)s"::json#>>'{%(keyf)s}')::int,
|
||||
'default')
|
||||
),
|
||||
false)
|
||||
WHERE ("%(jsonb_c)s"::json#>>'{%(keyf)s}')::text ~ '^[0-9\.]+$'
|
||||
AND ("%(jsonb_c)s"::json#>>'{%(keyf)s}')::int in (
|
||||
SELECT id_nomenclature FROM ref_nomenclatures.t_nomenclatures t
|
||||
JOIN ref_nomenclatures.bib_nomenclatures_types b USING (id_type)
|
||||
WHERE lower(b.mnemonique) = '%(keyf)s'
|
||||
)
|
||||
'''%{'keyf':keyfield,'tab':table,'sch':shema,'jsonb_c':jsonb_column})
|
||||
# print(sql)
|
||||
|
||||
with con.begin() as cnx:
|
||||
cnx.execute(sql)
|
||||
|
||||
def update_additional_key(
|
||||
con,table='cor_counting_occtax',shema='pr_occtax',jsonb_column='additional_fields',
|
||||
keyfield='Strate de végétation',new_keyfield='strate_flore'
|
||||
):
|
||||
sql = text('''
|
||||
UPDATE %(sch)s.%(tab)s
|
||||
SET "%(jsonb_c)s" = "%(jsonb_c)s" - '%(keyf)s' || jsonb_build_object(
|
||||
'%(new_keyf)s', "%(jsonb_c)s"->'%(keyf)s'
|
||||
)
|
||||
WHERE "%(jsonb_c)s" ? '%(keyf)s';
|
||||
'''%{'keyf':keyfield,'new_keyf':new_keyfield,'tab':table,'sch':shema,'jsonb_c':jsonb_column})
|
||||
# print(sql)
|
||||
|
||||
with con.begin() as cnx:
|
||||
cnx.execute(sql)
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from pycen import con_gn
|
||||
|
||||
|
||||
NAME_OUT = '/home/colas/Documents/tmp/v_bdc_status2.xlsx'
|
||||
FileIn = '/media/cgeier/SRV/FICHIERS/TRANSFERTS-EQUIPE/CG/BDC_STATUTS_15.xls'
|
||||
db = False
|
||||
|
||||
if db :
|
||||
sql = "SELECT * FROM taxonomie.v_bdc_status"
|
||||
df = pd.read_sql_query(sql,con_gn)
|
||||
if FileIn is not None :
|
||||
df = pd.read_excel(FileIn)
|
||||
df.columns = df.columns.str.lower()
|
||||
geo = ['Isère','Rhône-Alpes','France','France métropolitaine',np.nan]
|
||||
df = df[df.lb_adm_tr.isin(geo)]
|
||||
|
||||
|
||||
|
||||
df.drop(columns=['full_citation'],inplace=True)
|
||||
df.replace({
|
||||
r'[’]':"'",
|
||||
r'[àáâãäå]':'a',
|
||||
r'[èéêë]':'e',
|
||||
r'[ìíîï]':'i',
|
||||
r'[òóôõö]':'o',
|
||||
r'[ùúûü]':'u',
|
||||
r'[–]':"-"
|
||||
},regex=True,inplace=True)
|
||||
|
||||
DF = df.copy()
|
||||
|
||||
# ['cd_nom', 'cd_ref', 'rq_statut', 'code_statut', 'label_statut',
|
||||
# 'cd_type_statut', 'thematique', 'lb_type_statut', 'regroupement_type',
|
||||
# 'cd_st_text', 'cd_sig', 'cd_doc', 'niveau_admin', 'cd_iso3166_1',
|
||||
# 'cd_iso3166_2', 'doc_url', 'type_value']
|
||||
pivot = pd.pivot_table(
|
||||
DF,
|
||||
values='code_statut',
|
||||
index=['cd_nom', 'cd_ref'#,'niveau_admin','lb_adm_tr'
|
||||
],
|
||||
columns=['cd_type_statut'],
|
||||
aggfunc=list,fill_value=None)
|
||||
for c in pivot.columns:
|
||||
pivot[c] = [x[0] if x is not np.NaN and len(x)==1 else x for x in pivot[c]]
|
||||
pivot['DH'] = [','.join(x) if (x is not np.NaN) and (len(x)==2) else x for x in pivot['DH']]
|
||||
pivot.DH.replace({'CDH':''},regex=True,inplace=True)
|
||||
|
||||
pivlib = pd.pivot_table(
|
||||
DF,
|
||||
values='label_statut',
|
||||
index=['cd_nom', 'cd_ref'#,'niveau_admin','lb_adm_tr'
|
||||
],
|
||||
columns=['cd_type_statut'],
|
||||
aggfunc=list,fill_value=None)
|
||||
for c in pivlib.columns:
|
||||
pivlib[c] = [x[0] if x is not np.NaN and len(x)==1 else x for x in pivlib[c]]
|
||||
pivlib['DH'] = [','.join(x) if (x is not np.NaN) and (len(x)==2) else x for x in pivlib['DH']]
|
||||
pivlib.DH.replace({'CDH':''},regex=True,inplace=True)
|
||||
|
||||
print('INIT writer')
|
||||
with pd.ExcelWriter(NAME_OUT) as writer:
|
||||
DF.to_excel(
|
||||
writer,sheet_name='v_bdc_status'
|
||||
)
|
||||
# writer.save()
|
||||
print('v_bdc_status OK !')
|
||||
pivot.to_excel(
|
||||
writer,sheet_name='pivot_table'
|
||||
)
|
||||
# writer.save()
|
||||
print('pivot_table OK !')
|
||||
pivlib.to_excel(
|
||||
writer,sheet_name='pivot_libel'
|
||||
)
|
||||
writer.save()
|
||||
print('pivot_libel OK !')
|
||||
@@ -0,0 +1,473 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
|
||||
import requests
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import os
|
||||
|
||||
def get_status(lst,con):
|
||||
sql = """
|
||||
SELECT
|
||||
t.cd_nom,
|
||||
t.cd_ref,
|
||||
t.regne,
|
||||
t.phylum,
|
||||
t.classe,
|
||||
t.ordre,
|
||||
t.famille,
|
||||
t.group1_inpn,
|
||||
t.group2_inpn,
|
||||
t.group3_inpn,
|
||||
t.nom_vern,
|
||||
t.nom_complet,
|
||||
t.nom_valide,
|
||||
t.lb_nom,
|
||||
--s.*
|
||||
s.cd_sig,
|
||||
s.rq_statut,
|
||||
s.code_statut,
|
||||
s.cd_type_statut,
|
||||
s.label_statut,
|
||||
s.niveau_admin,
|
||||
s.full_citation,
|
||||
s.doc_url
|
||||
FROM taxonomie.taxref t
|
||||
LEFT JOIN taxonomie.v_bdc_status s USING (cd_nom)
|
||||
WHERE t.cd_nom IN {cd_nom}
|
||||
;""".format(cd_nom = tuple(lst))
|
||||
return pd.read_sql_query(sql,con)
|
||||
|
||||
def get_type_status(con):
|
||||
sql = """
|
||||
SELECT * FROM taxonomie.bdc_statut_type
|
||||
;"""
|
||||
return pd.read_sql_query(sql,con)
|
||||
|
||||
def get_api_status(api,cd_nom:int):
|
||||
res = requests.api.get('%s/%i'%(api,cd_nom))
|
||||
if res.status_code == 200:
|
||||
return res.json()
|
||||
else :
|
||||
raise('Error : %i\tcd_nom : %i'%(res.status_code,cd_nom))
|
||||
|
||||
def get_taxon_status(lst,api):
|
||||
from datetime import datetime as dt
|
||||
init = dt.now()
|
||||
st = [get_api_status(api,x) for x in lst] # TOO LONG
|
||||
print(dt.now()-init)
|
||||
phylo = {
|
||||
'cd_ref':[x['cd_ref'] for x in st],
|
||||
'nom_valide':[x['nom_valide'] if 'nom_valide' in x.keys() else None for x in st],
|
||||
'nom_vernac':[x['nom_vern'] if 'nom_vern' in x.keys() else None for x in st],
|
||||
'regne':[x['regne'] if 'regne' in x.keys() else None for x in st],
|
||||
'group1_inp':[x['group1_inpn'] if 'group1_inpn' in x.keys() else None for x in st],
|
||||
'group2_inp':[x['group2_inp'] if 'group2_inp' in x.keys() else None for x in st],
|
||||
'group3_inpn':[x['group3_inpn'] for x in st],
|
||||
'classe':[x['classe'] if 'classe' in x.keys() else None for x in st],
|
||||
'ordre':[x['ordre'] if 'ordre' in x.keys() else None for x in st],
|
||||
'famille':[x['famille'] if 'famille' in x.keys() else None for x in st]}
|
||||
cd_status = {
|
||||
'AL':[
|
||||
[val['values'][v]['code_statut']
|
||||
for val in x['status']['AL']['text'].values() for v in val['values'] ]
|
||||
if 'AL' in x['status'].keys() else None
|
||||
for x in st
|
||||
],
|
||||
'BERN':[
|
||||
[val['values'][v]['code_statut']
|
||||
for val in x['status']['BERN']['text'].values() for v in val['values'] ]
|
||||
if 'BERN' in x['status'].keys() else None
|
||||
for x in st
|
||||
],
|
||||
'BONN':[
|
||||
[val['values'][v]['code_statut']
|
||||
for val in x['status']['BONN']['text'].values() for v in val['values'] ]
|
||||
if 'BONN' in x['status'].keys() else None
|
||||
for x in st
|
||||
],
|
||||
'DH':[
|
||||
[val['values'][v]['code_statut']
|
||||
for val in x['status']['DH']['text'].values() for v in val['values'] ]
|
||||
if 'DH' in x['status'].keys() else None
|
||||
for x in st
|
||||
],
|
||||
'DO':[
|
||||
[val['values'][v]['code_statut']
|
||||
for val in x['status']['DO']['text'].values() for v in val['values'] ]
|
||||
if 'DO' in x['status'].keys() else None
|
||||
for x in st
|
||||
],
|
||||
'LRE':[
|
||||
[val['values'][v]['code_statut']
|
||||
for val in x['status']['LRE']['text'].values() for v in val['values'] ]
|
||||
if 'LRE' in x['status'].keys() else None
|
||||
for x in st
|
||||
],
|
||||
'LRM':[
|
||||
[val['values'][v]['code_statut']
|
||||
for val in x['status']['LRM']['text'].values() for v in val['values'] ]
|
||||
if 'LRM' in x['status'].keys() else None
|
||||
for x in st
|
||||
],
|
||||
'LRN':[
|
||||
[val['values'][v]['code_statut']
|
||||
for val in x['status']['LRN']['text'].values() for v in val['values'] ]
|
||||
if 'LRN' in x['status'].keys() else None
|
||||
for x in st
|
||||
],
|
||||
'LRR':[
|
||||
[val['values'][v]['code_statut']
|
||||
for val in x['status']['LRR']['text'].values() for v in val['values'] ]
|
||||
if 'LRR' in x['status'].keys() else None
|
||||
for x in st
|
||||
],
|
||||
'PAPNAT':[
|
||||
[val['values'][v]['code_statut']
|
||||
for val in x['status']['PAPNAT']['text'].values() for v in val['values'] ]
|
||||
if 'PAPNAT' in x['status'].keys() else None
|
||||
for x in st
|
||||
],
|
||||
'PD':[
|
||||
[val['values'][v]['code_statut']
|
||||
for val in x['status']['PD']['text'].values() for v in val['values'] ]
|
||||
if 'PD' in x['status'].keys() else None
|
||||
for x in st
|
||||
],
|
||||
'PNA':[
|
||||
[val['values'][v]['code_statut']
|
||||
for val in x['status']['PNA']['text'].values() for v in val['values'] ]
|
||||
if 'PNA' in x['status'].keys() else None
|
||||
for x in st
|
||||
],
|
||||
'PR':[
|
||||
[val['values'][v]['code_statut']
|
||||
for val in x['status']['PR']['text'].values() for v in val['values'] ]
|
||||
if 'PR' in x['status'].keys() else None
|
||||
for x in st
|
||||
],
|
||||
'REGL':[
|
||||
[val['values'][v]['code_statut']
|
||||
for val in x['status']['REGL']['text'].values() for v in val['values'] ]
|
||||
if 'REGL' in x['status'].keys() else None
|
||||
for x in st
|
||||
],
|
||||
'REGLII':[
|
||||
[val['values'][v]['code_statut']
|
||||
for val in x['status']['REGLII']['text'].values() for v in val['values'] ]
|
||||
if 'REGLII' in x['status'].keys() else None
|
||||
for x in st
|
||||
],
|
||||
'REGLLUTTE':[
|
||||
[val['values'][v]['code_statut']
|
||||
for val in x['status']['REGLLUTTE']['text'].values() for v in val['values'] ]
|
||||
if 'REGLLUTTE' in x['status'].keys() else None
|
||||
for x in st
|
||||
],
|
||||
'REGLSO':[
|
||||
[val['values'][v]['code_statut']
|
||||
for val in x['status']['REGLSO']['text'].values() for v in val['values'] ]
|
||||
if 'REGLSO' in x['status'].keys() else None
|
||||
for x in st
|
||||
],
|
||||
'SCAP NAT':[
|
||||
[val['values'][v]['code_statut']
|
||||
for val in x['status']['SCAP NAT']['text'].values() for v in val['values'] ]
|
||||
if 'SCAP NAT' in x['status'].keys() else None
|
||||
for x in st
|
||||
],
|
||||
'SCAP REG':[
|
||||
[val['values'][v]['code_statut']
|
||||
for val in x['status']['SCAP REG']['text'].values() for v in val['values'] ]
|
||||
if 'SCAP REG' in x['status'].keys() else None
|
||||
for x in st
|
||||
],
|
||||
'SENSNAT':[
|
||||
[val['values'][v]['code_statut']
|
||||
for val in x['status']['SENSNAT']['text'].values() for v in val['values'] ]
|
||||
if 'SENSNAT' in x['status'].keys() else None
|
||||
for x in st
|
||||
],
|
||||
'ZDET':[
|
||||
[val['values'][v]['code_statut']
|
||||
for val in x['status']['ZDET']['text'].values() for v in val['values'] ]
|
||||
if 'ZDET' in x['status'].keys() else None
|
||||
for x in st
|
||||
],
|
||||
'exPNA':[
|
||||
[val['values'][v]['code_statut']
|
||||
for val in x['status']['exPNA']['text'].values() for v in val['values'] ]
|
||||
if 'exPNA' in x['status'].keys() else None
|
||||
for x in st
|
||||
]
|
||||
}
|
||||
return pd.DataFrame({**phylo,**cd_status})
|
||||
|
||||
|
||||
def filter_bio_geo(df,zone_bio):
|
||||
idNotBioGeo = []
|
||||
# Filtre du dommaine biogeographique sur la plaine Rhodanienne et Alpine
|
||||
test_rhod = df.rq_statut.str.contains('rhodanienne : Non déterminante',na=False)
|
||||
test_alpi = df.rq_statut.str.contains('Alpine : Non déterminante',na=False)
|
||||
|
||||
if zone_bio == 'rhod':
|
||||
idNotBioGeo += (df[test_rhod].index.to_list())
|
||||
elif zone_bio == 'alpi':
|
||||
idNotBioGeo += (df[test_alpi].index.tolist())
|
||||
elif zone_bio == 'all':
|
||||
idNotBioGeo += (df[test_rhod&test_alpi].index.tolist())
|
||||
|
||||
if idNotBioGeo:
|
||||
df.iloc[idNotBioGeo,-6:] = None
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def form_territoire(df,terr):
|
||||
is_dep = df.cd_sig.str.contains('INSEED')
|
||||
dep = (
|
||||
['38','42','07','26'] if terr == 'platiere' else
|
||||
['38','69','01'] if terr == 'negria' else ['38']
|
||||
)
|
||||
keep_not = df[is_dep&(~df.cd_sig.str[-2:].isin(dep))].index
|
||||
df.drop(keep_not,inplace=True)
|
||||
|
||||
is_dep = df.cd_sig.str.contains('INSEED')
|
||||
is_38 = df.cd_sig=='INSEED38'
|
||||
|
||||
if terr == 'isere':
|
||||
filter_id = df[is_dep&(~is_38)].index
|
||||
df.drop(filter_id,inplace=True)
|
||||
else:
|
||||
lst_stat = df[is_dep&(~is_38)].cd_type_statut.unique()
|
||||
# is_lst = df.cd_type_statut.isin(lst_stat)
|
||||
df.loc[is_dep&(~is_38),'cd_type_statut'] = ( df[is_dep&(~is_38)]
|
||||
.cd_type_statut + '_' + df[is_dep&(~is_38)].cd_sig.str.strip('INSEED')
|
||||
)
|
||||
df.loc[is_38,'cd_type_statut'] = ( df[is_38]
|
||||
.cd_type_statut + '_38'
|
||||
)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def complete_tax_missing(test,tax,cd):
|
||||
for c in cols_taxonomie:
|
||||
if (c in test.columns) & (test[c].isna().any()) :
|
||||
test.loc[test[c].isna(),c] = [tax[tax[cd]==i][c].values[0] for i in test[test[c].isna()].index]
|
||||
|
||||
return test
|
||||
|
||||
|
||||
dict_dep = {
|
||||
'38':'Isère',
|
||||
'42':'Loire',
|
||||
'07':'Ardèche',
|
||||
'26':'Drôme',
|
||||
}
|
||||
|
||||
|
||||
cols_rename = {
|
||||
'nom_vernaculaire': 'nom_vern'
|
||||
}
|
||||
|
||||
cols_taxonomie = [
|
||||
'cd_nom', 'cd_ref', 'regne', 'phylum', 'classe', 'ordre', 'famille',
|
||||
'group1_inpn', 'group2_inpn', 'group3_inpn', 'nom_vern', 'nom_complet',
|
||||
'nom_valide'
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Définition de la connection à la bdd GéoNature
|
||||
from pycen import con_gn
|
||||
# NOT USE FOR NOW - API Taxref
|
||||
api_taxref = 'https://geonature.cen-isere.fr/taxhub/api/taxref'
|
||||
|
||||
# Paramètres de chargement du fichier des taxons
|
||||
PATH0 = '/media/cgeier/SRV/FICHIERS/OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/SITES/HUEZ/Données espèces/'
|
||||
PATH = ''
|
||||
file = 'Synthèse taxon - sources aggregées (point).xlsx'
|
||||
sheet = 'Sheet1'
|
||||
zone_bio_znieff = 'alpi' # ['rhod', 'alpi', 'all']
|
||||
territoire = 'isere' # ['isere', 'platiere','negria']
|
||||
# [GEOMETRY PARAMS]
|
||||
keep_geomtype = 'point' # ['polygon', 'point', 'ligne', None]
|
||||
geom_col = 'geometry'
|
||||
|
||||
# Liste des CD_NOM en entrée
|
||||
cd_col = 'cd_ref' # Nom de la colonne à utiliser dans le feuillet ``sheet``
|
||||
|
||||
|
||||
# Lecture des données
|
||||
taxlist = pd.read_excel(os.path.join(PATH0,PATH,file),sheet,usecols=[cd_col],header=0)
|
||||
tab_sp = pd.read_excel(os.path.join(PATH0,PATH,file),sheet,index_col=cd_col)
|
||||
tab_sp.rename(columns=cols_rename,inplace=True)
|
||||
|
||||
# Exclusion d'un type de géométrie
|
||||
if keep_geomtype is not None:
|
||||
import geopandas as gpd
|
||||
|
||||
as_geom = tab_sp[geom_col].str.contains(keep_geomtype.upper(),na=False)
|
||||
tab_sp = tab_sp[as_geom]
|
||||
tab_sp[geom_col] = gpd.GeoSeries.from_wkt(tab_sp.geometry, crs=4326)
|
||||
tab_sp = tab_sp.set_geometry(geom_col)
|
||||
taxlist = taxlist[taxlist[cd_col].isin(tab_sp.index.tolist())]
|
||||
tax_geo_union = tab_sp.dissolve(by=tab_sp.index)[[geom_col]]
|
||||
|
||||
lst = taxlist[cd_col]
|
||||
|
||||
# Récupération des statuts
|
||||
df = get_status(lst.astype(str).unique(),con_gn)
|
||||
typ = get_type_status(con_gn)
|
||||
typ = typ[typ.cd_type_statut.isin(df.cd_type_statut.unique())]
|
||||
|
||||
# Distinction LRR [old vs new] région
|
||||
is_lrr = df.cd_type_statut == 'LRR'
|
||||
df.loc[is_lrr & (df.niveau_admin == 'Région'),'cd_type_statut'] = 'LRR_AURA'
|
||||
df.loc[is_lrr & (df.niveau_admin == 'Ancienne région'),'cd_type_statut'] = 'LRR_RA'
|
||||
del df['niveau_admin']
|
||||
|
||||
# Filtre du dommaine biogeographique sur la plaine Rhodanienne et Alpine
|
||||
df = filter_bio_geo(df,zone_bio_znieff)
|
||||
|
||||
# Filtre des statuts vis à vis du territoire d'intérêts
|
||||
# Conservation des statuts Adrèche, Drôme et Loire pour la Platière
|
||||
df = form_territoire(df,territoire)
|
||||
|
||||
# for c in ['cd_ref','cd_nom','lb_nom']:
|
||||
# if c in tab_sp.columns:
|
||||
# # if 'cd_nom' not in df.columns and c == 'cd_ref': continue
|
||||
# tab_sp.drop(c,axis=1,inplace=True)
|
||||
|
||||
keep_cols = same_col = df.columns[df.columns.isin(tab_sp.reset_index(drop=False).columns)]
|
||||
as_vern = keep_cols.str.contains('vern').any()
|
||||
as_numb = keep_cols.str.contains('nombre').any()
|
||||
if not as_vern:
|
||||
keep_cols = [*keep_cols,'nom_vern']
|
||||
if not as_numb:
|
||||
keep_cols = [*keep_cols,'nombre_min','nombre_max']
|
||||
else:
|
||||
same_col = [x for x in keep_cols if x not in ['nombre_min','nombre_max']]
|
||||
|
||||
piv = pd.pivot_table(
|
||||
df,
|
||||
values='code_statut',
|
||||
index=['cd_nom', 'cd_ref','lb_nom','nom_vern'#,'niveau_admin','lb_adm_tr'
|
||||
],
|
||||
columns=['cd_type_statut'],
|
||||
aggfunc=list,fill_value=None)
|
||||
|
||||
for c in piv.columns:
|
||||
piv[c] = [x[0] if x is not np.NaN and len(x)==1 else x for x in piv[c]]
|
||||
if 'DH' in piv.columns:
|
||||
piv['DH'] = [','.join(x) if (x is not np.NaN) and (len(x)==2) else x for x in piv['DH']]
|
||||
piv.DH.replace({'CDH':''},regex=True,inplace=True)
|
||||
# piv.reset_index(-1, inplace=True)
|
||||
|
||||
# complete_tax_missing(tab_sp,df,cd_col)
|
||||
group_sp = (
|
||||
tab_sp.reset_index(drop=False)[[cd_col,'nombre_min','nombre_max','date_debut','date_fin']]
|
||||
.groupby([cd_col])
|
||||
.agg({
|
||||
'nombre_min':'sum',
|
||||
'nombre_max':'sum',
|
||||
'date_debut':'min',
|
||||
'date_fin':'max'
|
||||
})
|
||||
.astype({
|
||||
'nombre_min':int,
|
||||
'nombre_max':int,
|
||||
})
|
||||
)
|
||||
|
||||
# tabsp_nb = tab_sp.reset_index(drop=False)[[*same_col,'nombre_min','nombre_max']].groupby([*same_col]).sum()
|
||||
tabsp_nb = (
|
||||
# tab_sp.sort_index().reset_index(drop=False)[same_col]#[[cd_col]]
|
||||
df[same_col]
|
||||
.drop_duplicates()
|
||||
# .merge(df[cols_taxonomie].drop_duplicates(), on = cd_col, how='left')
|
||||
.merge(group_sp, on = cd_col)
|
||||
)
|
||||
tabsp_nb.set_index(cd_col,inplace=True)
|
||||
tabsp_nb.sort_index(inplace=True)
|
||||
|
||||
# nb_missing = df[~df.cd_nom.isin(tabsp_nb.index)].cd_nom.unique()
|
||||
# nb_missing = tabsp_nb[~tabsp_nb.index.isin(df.cd_nom)].index.unique()
|
||||
# nb_missing = tab_sp[~tab_sp.index.isin(tabsp_nb.index)].index.unique()
|
||||
# tabsp_nb.merge(group_sp[group_sp.index.isin(nb_missing)], right_index = True, left_on = 'cd_nom')
|
||||
|
||||
# pivot = tab_sp.merge(piv,on=[cd_col],how='left')
|
||||
pivot = tabsp_nb.merge(piv,on=[cd_col],how='left')
|
||||
|
||||
pivlib = pd.pivot_table(
|
||||
df,
|
||||
values='label_statut',
|
||||
index=[
|
||||
'cd_nom', 'cd_ref','lb_nom','nom_vern'#,'niveau_admin','lb_adm_tr'
|
||||
],
|
||||
columns=['cd_type_statut'],
|
||||
aggfunc=list,fill_value=None)
|
||||
for c in pivlib.columns:
|
||||
pivlib[c] = [x[0] if x is not np.NaN and len(x)==1 else x for x in pivlib[c]]
|
||||
if 'DH' in pivot.columns:
|
||||
pivlib['DH'] = [','.join(x) if (x is not np.NaN) and (len(x)==2) else x for x in pivlib['DH']]
|
||||
pivlib.DH.replace({'CDH':''},regex=True,inplace=True)
|
||||
|
||||
# pivlib = tab_sp.merge(pivlib,on=[cd_col],how='left')
|
||||
pivotlib = tabsp_nb.merge(pivlib,on=[cd_col],how='left')
|
||||
|
||||
tx_nb_missing = df[~df.cd_nom.isin(tabsp_nb.index)].shape[0]
|
||||
tx_nb_missing = df[~df[cd_col].isin(tabsp_nb.index)].shape[0]
|
||||
if tx_nb_missing > 0:
|
||||
print('WARNING : %i taxon(s) is MISING !! \n'%tx_nb_missing)
|
||||
|
||||
if keep_geomtype:
|
||||
|
||||
print('INIT GeoPackage writer')
|
||||
NAME_OUT = os.path.join('~/','Statuts '+file.split('.')[0]+'.gpkg')
|
||||
(pivot
|
||||
.merge(tax_geo_union,left_index=True,right_index=True)
|
||||
.set_geometry(geom_col,crs=4326)
|
||||
.to_file(
|
||||
NAME_OUT, layer='Statut_code', driver='GPKG'
|
||||
)
|
||||
)
|
||||
(pivotlib
|
||||
.merge(tax_geo_union,left_index=True,right_index=True)
|
||||
.set_geometry(geom_col,crs=4326)
|
||||
.to_file(
|
||||
NAME_OUT, layer='Statut_label', driver='GPKG'
|
||||
)
|
||||
)
|
||||
|
||||
else:
|
||||
print('INIT Excel writer')
|
||||
NAME_OUT = os.path.join('~/',sheet+'_status.xlsx')
|
||||
if keep_geomtype is not None:
|
||||
NAME_OUT = NAME_OUT[:-5]+' (%s only).xlsx'%keep_geomtype.lower()
|
||||
|
||||
with pd.ExcelWriter(NAME_OUT) as writer:
|
||||
df.to_excel(
|
||||
writer,sheet_name='v_bdc_status',index=False
|
||||
)
|
||||
# writer.save()
|
||||
print('v_bdc_status OK !')
|
||||
pivot.to_excel(
|
||||
writer,sheet_name='pivot_table'
|
||||
)
|
||||
# writer.save()
|
||||
print('pivot_table OK !')
|
||||
pivotlib.to_excel(
|
||||
writer,sheet_name='pivot_libel'
|
||||
)
|
||||
# writer.save()
|
||||
print('pivot_libel OK !')
|
||||
typ.to_excel(
|
||||
writer,sheet_name='dic_type_statut',index=False
|
||||
)
|
||||
# writer.save()
|
||||
print('dic_type_statut OK !')
|
||||
|
||||
print('END writing %s'%NAME_OUT)
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
|
||||
from owslib.wfs import WebFeatureService
|
||||
import geopandas as gpd
|
||||
from geoalchemy2 import Geometry
|
||||
from sqlalchemy import dialects
|
||||
|
||||
|
||||
|
||||
def get_wfs(url, layer, bbox=None):
|
||||
from geopandas import read_file
|
||||
from requests import Request
|
||||
|
||||
wfs = WebFeatureService(url=url)
|
||||
item = dict(wfs.items())[layer]
|
||||
crs = item.crsOptions[0].getcode()
|
||||
|
||||
params = dict(service='WFS', version=wfs.version, request='GetFeature',
|
||||
typeName=layer)
|
||||
q = Request('GET', url, params=params).prepare().url
|
||||
|
||||
data = read_file(q, bbox=bbox)
|
||||
data.set_crs(crs=crs, inplace=True)
|
||||
|
||||
if crs != 'EPSG:2154':
|
||||
data.to_crs(epsg=2154, inplace=True)
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def list_layer(url):
|
||||
wfs = WebFeatureService(url=url)
|
||||
lst = list(wfs.contents)
|
||||
return lst
|
||||
|
||||
|
||||
def Polygons_to_MultiPolygon(df,geom_col=None):
|
||||
from shapely.geometry import MultiPolygon
|
||||
from pandas import concat
|
||||
|
||||
if not geom_col:
|
||||
geom_col = df.geometry.name
|
||||
|
||||
df = df.copy()
|
||||
multi = df.loc[df.geom_type=='MultiPolygon'].copy()
|
||||
poly = df.loc[df.geom_type=='Polygon'].copy()
|
||||
poly[geom_col] = [MultiPolygon([geom]) for geom in df.loc[df.geom_type=='Polygon',geom_col] ]
|
||||
df = concat([multi,poly])
|
||||
df.sort_index(inplace=True)
|
||||
return df
|
||||
|
||||
|
||||
def ref_geo(type_code,con):
|
||||
sql = """
|
||||
SELECT l.* FROM ref_geo.l_areas l
|
||||
JOIN ref_geo.bib_areas_types bib USING (id_type)
|
||||
WHERE bib.type_code='{typ}'
|
||||
""".format(typ=type_code)
|
||||
return gpd.read_postgis(sql,con)
|
||||
|
||||
|
||||
def to_lareas(df,dic,layer,con,dtypes={}):
|
||||
|
||||
id_type = gpd.pd.read_sql_query("""
|
||||
SELECT id_type FROM ref_geo.bib_areas_types WHERE type_code='%s'
|
||||
"""%layer,con).id_type.values[0]
|
||||
|
||||
ref = ref_geo(layer,con)
|
||||
|
||||
df.rename(columns=dic, inplace=True)
|
||||
df = Polygons_to_MultiPolygon(
|
||||
df[~df.area_name.isin(ref.area_name)]
|
||||
)
|
||||
df.rename_geometry('geom', inplace=True)
|
||||
del_col = df.columns[~df.columns.isin(['geom',*[*set(dic.values())]])]
|
||||
df.drop(columns=del_col, inplace=True)
|
||||
|
||||
df['id_type'] = id_type
|
||||
df['geojson_4326'] = df.to_crs(4326).geom.__geo_interface__['features']
|
||||
df['geojson_4326'] = [x['geometry'] for x in df['geojson_4326']]
|
||||
df['centroid'] = 'SRID=2154;'+df.geom.centroid.to_wkt()
|
||||
df['enable'] = True
|
||||
|
||||
if df.empty:
|
||||
print('AUCUN nouveaux zonages identifiés')
|
||||
else:
|
||||
df.to_postgis(
|
||||
name='l_areas',
|
||||
con=con,
|
||||
schema='ref_geo',
|
||||
if_exists='append',
|
||||
index=False,
|
||||
index_label=None,
|
||||
chunksize=None,
|
||||
dtype={
|
||||
'centroid': Geometry(geometry_type='POINT',srid=2154),
|
||||
**dtypes
|
||||
},
|
||||
)
|
||||
print('INSERT %i zones'%df.shape[0])
|
||||
|
||||
|
||||
|
||||
def update_larea(con,layer,df,cols_updt=[],on='area_name'):
|
||||
from pycen import update_to_sql
|
||||
table = 'l_areas'
|
||||
# idtyp = tuple(df.id_type.unique())
|
||||
|
||||
idtyp = gpd.pd.read_sql_query("""
|
||||
SELECT id_type FROM ref_geo.bib_areas_types WHERE type_code='%s'
|
||||
"""%layer,con).id_type.values[0]
|
||||
|
||||
# if len(idtyp) > 1:
|
||||
# where = 'in %s'%tuple(df.id_type.unique())
|
||||
# else :
|
||||
where = '=%i'%idtyp
|
||||
|
||||
sql = 'SELECT id_area, %s FROM ref_geo.l_areas WHERE id_type %s;'%(on,where)
|
||||
larea = gpd.pd.read_sql_query(sql,con)
|
||||
|
||||
to_updt = df.merge(larea, on=on)
|
||||
update_to_sql(
|
||||
df=to_updt[['id_area',*cols_updt]],
|
||||
con=con,
|
||||
table_name=table,
|
||||
schema_name='ref_geo',
|
||||
key_name='id_area',
|
||||
)
|
||||
|
||||
|
||||
carmen = 'https://ws.carmencarto.fr/WFS/119/fxx_inpn?'
|
||||
dic_layer = {
|
||||
# ref_geo.bib_areas_types.type_code : layer_name from list_layer
|
||||
'ZNIEFF2':'Znieff2',
|
||||
'ZNIEFF1':'Znieff1',
|
||||
'APB':'Arretes_de_protection_de_biotope',
|
||||
'RNN':'Reserves_naturelles_nationales',
|
||||
'RNR':'Reserves_naturelles_regionales',
|
||||
'ZPS':'Zones_de_protection_speciale',
|
||||
'SIC':'Sites_d_importance_communautaire',
|
||||
'ZICO':'ZICO',
|
||||
'RNCFS':'Reserves_nationales_de_chasse_et_faune_sauvage',
|
||||
'RIPN':'Reserves_Integrales_de_Parcs_Nationaux', # NO zone in 74
|
||||
'SCEN':'Terrains_acquis_des_Conservatoires_des_espaces_naturels', # Pas fait ;
|
||||
# 'SCL':'',
|
||||
# 'PNM':'',
|
||||
'PNR':'Parcs_naturels_regionaux',
|
||||
'RBIOL':'Reserves_biologiques',
|
||||
'RBIOS':'Reserves_de_la_biosphere', # NO zone in 74
|
||||
# 'RNC':'',
|
||||
'SRAM':'Sites_Ramsar',
|
||||
# 'AA':'',
|
||||
# 'ZSC':'',
|
||||
# 'PSIC':'',
|
||||
# 'PEC':'',
|
||||
# ...
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
from sqlalchemy import create_engine # pour lecture de la bd Géonature
|
||||
from sqlalchemy.engine import URL
|
||||
# Iddentifiant de connexion au GeoNature du CEN 74
|
||||
usr = 'XXXXX'
|
||||
pdw='XXXX'
|
||||
host='XXXXX'
|
||||
bd ='XXXXX'
|
||||
|
||||
eng = URL.create('postgresql+psycopg2',username=usr,password=pdw,host=host,database=bd)
|
||||
conn = create_engine(eng)
|
||||
# from pycen import con_gn as conn
|
||||
|
||||
ref = ref_geo('DEP',conn)
|
||||
ter = ref[ref.area_code=='74']
|
||||
|
||||
layer = 'Znieff1'
|
||||
# layer_name = dic_layer[layer]
|
||||
zon = get_wfs(carmen,layer,bbox=ter.unary_union)
|
||||
zon = zon[zon.intersects(ter.unary_union)]
|
||||
|
||||
dic_cols = {
|
||||
'ID_MNHN':'area_code',
|
||||
'NOM':'area_name',
|
||||
'URL':'source',
|
||||
}
|
||||
|
||||
dtyp = {}
|
||||
if layer in ['ZPS','SIC']:
|
||||
dic_cols = {**dic_cols,**{'SITECODE':'area_code','SITENAME':'area_name',}}
|
||||
elif layer == 'RNN':
|
||||
dic_cols = {**dic_cols,**{'URL_FICHE':'source','NOM_SITE':'area_name',}}
|
||||
elif layer == 'ZICO':
|
||||
dic_cols = {**dic_cols,**{'ID_SPN':'area_code'}}
|
||||
elif layer == 'SCEN':
|
||||
zon.NOM = zon.NOM.str.title()
|
||||
elif layer == 'RBIOL':
|
||||
dic_cols = {
|
||||
**dic_cols,
|
||||
**{'URL_FICHE':'source','NOM_SITE':'area_name',
|
||||
'comment':'comment','additional_data':'additional_data',}
|
||||
}
|
||||
zon['comment'] = 'Réserves biologiques ' + zon.CODE_R_ENP.replace(
|
||||
{'I': 'intégrale','D' : 'dirigée'}
|
||||
)
|
||||
zon['additional_data'] = [
|
||||
str({ zon.CODE_R_ENP.name : x }).replace("\'",'\"')
|
||||
for x in zon.CODE_R_ENP
|
||||
]
|
||||
dtyp = {'additional_data':dialects.postgresql.JSONB}
|
||||
|
||||
|
||||
# IF UPDATE DATA
|
||||
# update_larea(conn,layer,zon.rename(columns=dic_cols),['comment'],on='area_code')
|
||||
|
||||
|
||||
to_lareas(
|
||||
df=zon,
|
||||
dic=dic_cols,
|
||||
layer=layer,
|
||||
con=conn,
|
||||
dtypes=dtyp
|
||||
)
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
import geopandas as gpd
|
||||
from pycen import con_gn
|
||||
|
||||
uuid_gn38 = gpd.pd.read_sql('SELECT unique_id_sinp uuid_perm_sinp FROM gn_synthese.synthese',con_gn)
|
||||
lst_gn38 = uuid_gn38.uuid_perm_sinp.astype(str).tolist()
|
||||
|
||||
PATH = '/media/cgeier/SRV/FICHIERS'
|
||||
biodiv = gpd.read_file(PATH+'/OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/SITES/Lone de la Negria/Faune-Flore/Données partenaires/0.Données transmises/BiodivAura_Synthese_Export_2026-04-21.geojson')
|
||||
cenra_point = gpd.read_file(PATH+'/OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/SITES/Lone de la Negria/Faune-Flore/Données partenaires/0.Données transmises/CENRA_20260421_gn_export_lones_cen38.gpkg',layer = 'points')
|
||||
cenra_poly = gpd.read_file(PATH+'/OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/SITES/Lone de la Negria/Faune-Flore/Données partenaires/0.Données transmises/CENRA_20260421_gn_export_lones_cen38.gpkg',layer = 'polygones')
|
||||
# synt = gpd.pd.read_csv(PATH+'/OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/SITES/Lone de la Negria/Faune-Flore/Données partenaires/0.Données transmises/synthese_observations_2025-04-24T12_11_45.488Z (completer).csv',sep=',')
|
||||
|
||||
# synt['code_com'] = synt.communes.str.split('(',expand=True)[1].str[:5]
|
||||
# synt['code_dep'] = synt.communes.str.split('(',expand=True)[1].str[:2]
|
||||
# synt.nombre_min = synt.nombre_min.astype(str).str.strip('.0').replace('nan',None)
|
||||
# synt.nombre_max = synt.nombre_max.astype(str).str.strip('.0').replace('nan',None)
|
||||
# synt.precision_geographique = synt.precision_geographique.astype(str).str.strip('.0').replace('nan',None)
|
||||
# synt.alti_min = synt.alti_min.astype(str).str.strip('.0').replace('nan',None)
|
||||
|
||||
for cenra in [cenra_point,cenra_poly]:
|
||||
test_nb_cenra = cenra.nombre_min.astype('float') > cenra.nombre_max.astype('float')
|
||||
if test_nb_cenra.any():
|
||||
cenra.loc[test_nb_cenra,'nb_tmp'] = cenra[test_nb_cenra].nombre_min
|
||||
cenra.loc[test_nb_cenra,'nombre_min'] = cenra[test_nb_cenra].nombre_max
|
||||
cenra.loc[test_nb_cenra,'nombre_max'] = cenra[test_nb_cenra].nb_tmp
|
||||
cenra.drop(columns='nb_tmp',inplace=True)
|
||||
|
||||
# cenra.nombre_min = cenra.nombre_min.astype(str).str.strip('.0').replace('nan',None)
|
||||
# cenra.nombre_max = cenra.nombre_max.astype(str).str.strip('.0').replace('nan',None)
|
||||
cenra.profondeur_min = cenra.profondeur_min.astype(str).str.strip('.0').replace('nan',None)
|
||||
cenra.profondeur_max = cenra.profondeur_max.astype(str).str.strip('.0').replace('nan',None)
|
||||
cenra.precision = cenra.precision.astype(str).str.strip('.0').replace('nan',None)
|
||||
cenra.code_habitat = cenra.code_habitat.astype(str).str.strip('.0').replace('nan',None)
|
||||
# cenra.altitude_min = cenra.altitude_min.astype(str).str.strip('.0').replace('nan',None)
|
||||
# cenra.altitude_max = cenra.altitude_max.astype(str).str.strip('.0').replace('nan',None)
|
||||
cenra.precision_diffusion.fillna('Non déterminé', inplace=True)
|
||||
|
||||
|
||||
# synt_l93 = synt[synt.x_centroid_4326>500000]
|
||||
# synt_l93_geom = gpd.GeoDataFrame(synt_l93,geometry=gpd.geoseries.points_from_xy(synt_l93.x_centroid_4326,synt_l93.y_centroid_4326),crs=2154)
|
||||
# synt_l93_geom.to_crs(4326,inplace=True)
|
||||
# synt_l93_geom.x_centroid_4326 = synt_l93_geom.geometry.x
|
||||
# synt_l93_geom.y_centroid_4326 = synt_l93_geom.geometry.y
|
||||
# synt_w84 = synt[synt.x_centroid_4326<500000]
|
||||
# synt = gpd.pd.concat([synt_l93_geom,synt_w84])
|
||||
|
||||
|
||||
|
||||
biodiv.nombre_min = biodiv.nombre_min.astype(str).str.strip('.0').replace('nan',None).replace('',None)
|
||||
biodiv.nombre_max = biodiv.nombre_max.astype(str).str.strip('.0').replace('nan',None).replace('',None)
|
||||
biodiv.alti_min = biodiv.alti_min.astype(str).str.strip('.0').replace('nan',None)
|
||||
biodiv.precision_geographique = biodiv.precision_geographique.astype(str).str.strip('.0').replace('nan',None)
|
||||
biodiv.observateurs.fillna('Inconnu', inplace=True)
|
||||
biodiv.stade_vie.fillna('Inconnu', inplace=True)
|
||||
biodiv.loc[biodiv.cd_ref == 1078938, 'cd_ref'] = 5289 # Geheebia lurida (Hornsch. ex Spreng.) J.A.Jiménez & M.J.Cano, 2021
|
||||
biodiv.loc[biodiv.cd_ref == 1049429, 'cd_ref'] = 108898 # Muscari neglectum Guss. ex Ten. & Sangiov., 1841
|
||||
|
||||
test_nb_biodiv = biodiv.nombre_min.astype('float') > biodiv.nombre_max.astype('float')
|
||||
if test_nb_biodiv.any():
|
||||
biodiv.loc[test_nb_biodiv,'nb_tmp'] = biodiv[test_nb_biodiv].nombre_min
|
||||
biodiv.loc[test_nb_biodiv,'nombre_min'] = biodiv[test_nb_biodiv].nombre_max
|
||||
biodiv.loc[test_nb_biodiv,'nombre_max'] = biodiv[test_nb_biodiv].nb_tmp
|
||||
biodiv.drop(columns='nb_tmp',inplace=True)
|
||||
|
||||
lst_cenra = [
|
||||
*cenra_point.id_perm_sinp.tolist(),
|
||||
*cenra_poly.id_perm_sinp.tolist()
|
||||
]
|
||||
|
||||
# synt = synt[~synt.uuid_perm_sinp.isin(lst_cenra)]
|
||||
biodiv = biodiv[~biodiv.uuid_perm_sinp.isin(lst_cenra)]
|
||||
biodiv = biodiv[~biodiv.uuid_perm_sinp.isin(lst_cenra)]
|
||||
|
||||
# lst_synt = synt.uuid_perm_sinp.tolist()
|
||||
lst_biodiv = biodiv.uuid_perm_sinp.tolist()
|
||||
|
||||
# biodiv = biodiv[~biodiv.uuid_perm_sinp.isin(lst_synt)]
|
||||
biodiv = biodiv[~biodiv.uuid_perm_sinp.isin(lst_gn38)]
|
||||
# synt[~synt.uuid_perm_sinp.isin(lst_biodiv)]
|
||||
|
||||
from shapely.geometry import shape
|
||||
import json
|
||||
biodiv['geom'] = biodiv.apply(lambda row: shape(json.loads(row['geojson_4326'])), axis=1).set_crs(4326)
|
||||
biodiv.set_geometry('geom',inplace=True)
|
||||
biodiv.drop(columns='geometry',inplace=True)
|
||||
biodiv['geom_4326'] = biodiv.geom.to_wkt()
|
||||
|
||||
gpd.pd.concat([cenra_point,cenra_poly]).to_csv(PATH+'/OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/SITES/Lone de la Negria/Faune-Flore/Données partenaires/1.Données nettoyées/CENRA_20260421_gn_export_lones_cen38.csv',index=False)
|
||||
# cenra.to_csv(PATH+'/OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/SITES/Lone de la Negria/Faune-Flore/Données partenaires/1.Données nettoyées/CENRA_synthese_observations_geonature_2026-03-16T15_25_59.351Z.csv',index=False)
|
||||
# synt.to_csv(PATH+'/OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/SITES/Lone de la Negria/Faune-Flore/Données partenaires/1.Données nettoyées/synthese_observations_2025-04-24T12_11_45.488Z.csv',index=False)
|
||||
biodiv.to_csv(PATH+'/OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/SITES/Lone de la Negria/Faune-Flore/Données partenaires/1.Données nettoyées/BiodivAura_Synthese_Export_2026-04-21.csv',index=False)
|
||||
|
||||
gpd.pd.concat([cenra_point,cenra_poly]).to_file(PATH+'/OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/SITES/Lone de la Negria/Faune-Flore/Données partenaires/1.Données nettoyées/CENRA_20260421_gn_export_lones_cen38.geojson',index=False)
|
||||
# cenra.to_file(PATH+'/OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/SITES/Lone de la Negria/Faune-Flore/Données partenaires/1.Données nettoyées/CENRA_synthese_observations_geonature_2026-03-16T15_25_59.351Z.geojson',index=False)
|
||||
# synt.to_file(PATH+'/OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/SITES/Lone de la Negria/Faune-Flore/Données partenaires/1.Données nettoyées/synthese_observations_2025-04-24T12_11_45.488Z.geojson',index=False)
|
||||
# synt_geom = gpd.GeoDataFrame(synt,geometry=gpd.geoseries.points_from_xy(synt.x_centroid_4326,synt.y_centroid_4326),crs=4326)
|
||||
# synt_geom.to_file(PATH+'/OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/SITES/Lone de la Negria/Faune-Flore/Données partenaires/1.Données nettoyées/synthese_observations_2025-04-24T12_11_45.488Z.geojson',index=False)
|
||||
biodiv.to_file(PATH+'/OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/SITES/Lone de la Negria/Faune-Flore/Données partenaires/1.Données nettoyées/BiodivAura_Synthese_Export_2026-04-21.geojson',index=False)
|
||||
@@ -0,0 +1,8 @@
|
||||
from pycen import con_gn
|
||||
import pandas as pd
|
||||
|
||||
tab = pd.read_excel('/home/colas/Documents/9_PROJETS/6_GEONATURE/citizen/liste_reptiles_rngl.xlsx')
|
||||
|
||||
sql = 'select cd_nom,lb_nom from taxonomie.taxref where lb_nom in {}'.format(tuple(tab['lb_nom']))
|
||||
res = pd.read_sql(sql,con_gn)
|
||||
lst_cdnom = res.cd_nom.astype(str).tolist()
|
||||
Reference in New Issue
Block a user