Compare commits

..

No commits in common. "8235e10773e96f798bbccf30990c1e6ae9f72d70" and "ca6c074aca5fbc8061a4f51bbcd93e809a715f3b" have entirely different histories.

10 changed files with 53 additions and 569 deletions

File diff suppressed because one or more lines are too long

View File

@ -1,29 +0,0 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
sql = '''
SELECT
split_part(b.data_id,':',2)::uuid data_id,
"submitterName" digitiser,
autre_catego_sensi,
autre_type_sanct,
catego_controle,
catego_sensibil,
"Commentaire_remarque_nombre_de_tentes",
"Conformit_du_contr_le",
day "date",
"heure",
infract_nb,
nat_operation,
nature_pers,
nb_pers_control,
nb_pers_sensibil,
nb_verb,
nom_agent,
type_sanct,
"submissionDate" date_saisie,
"updatedAt" date_maj,
st_force2d(st_geomfromgeojson(replace(COALESCE(geom,point_carte),'\','')))::geometry(geometry, 4326) AS geom
FROM police.infraction_rnn_submissions_data AS b
'''

View File

@ -1,177 +0,0 @@
from pycen import con_gn
sql = '''
DROP VIEW IF EXISTS gn_exports.v_synthese_sinp_with_metadata_flora_for_gn2pg_bis;
CREATE VIEW gn_exports.v_synthese_sinp_with_metadata_flora_for_gn2pg_bis AS
WITH af_actors AS (
SELECT cafa.id_acquisition_framework
FROM gn_meta.cor_acquisition_framework_actor cafa
LEFT JOIN utilisateurs.bib_organismes borg ON cafa.id_organism = borg.id_organisme
WHERE borg.uuid_organisme = '5a433bd0-2070-25d9-e053-2614a8c026f8'::uuid
), af AS (
SELECT taf.id_acquisition_framework
FROM gn_meta.t_acquisition_frameworks taf
JOIN af_actors ON af_actors.id_acquisition_framework = taf.id_acquisition_framework
GROUP BY taf.id_acquisition_framework
), ds AS (
SELECT tds.id_dataset,
tds.dataset_name AS nom_jdd,
tds.id_acquisition_framework
FROM gn_meta.t_datasets tds
GROUP BY tds.id_dataset, tds.dataset_name
), geo AS (
SELECT "left"(geo_1.area_code::text, 2) AS departement,
geo_1.area_code AS commune,
s_1.id_synthese,
st_transform(s_1.the_geom_local, 4326) AS st_transform
FROM ref_geo.l_areas geo_1
JOIN gn_synthese.synthese s_1 ON st_intersects(s_1.the_geom_4326, st_transform(geo_1.geom, 4326))
WHERE geo_1.id_type = 25
)
SELECT
ds.nom_jdd AS nom_jdd,
s.unique_id_sinp_grp AS id_sinp_releve,
occ.id_releve_occtax AS identifiant_releve,
NULL::text AS code_perso_releve,
s.unique_id_sinp AS id_sinp_observation,
s.entity_source_pk_value AS identifiant_observation,
geo.departement,
geo.commune,
NULL::text AS lieu_dit,
CASE
WHEN "position"(sp.srtext::text, 'GEOGCS'::text) = 1 THEN "substring"(replace(sp.srtext::text, 'GEOGCS["'::text, ''::text), 1, "position"(replace(sp.srtext::text, 'GEOGCS["'::text, ''::text), '",'::text) - 1)
WHEN "position"(sp.srtext::text, 'PROJCS'::text) = 1 THEN "substring"(replace(sp.srtext::text, 'PROJCS["'::text, ''::text), 1, "position"(replace(sp.srtext::text, 'PROJCS["'::text, ''::text), '",'::text) - 1)
WHEN "position"(sp.srtext::text, 'GEOCCS'::text) = 1 THEN "substring"(replace(sp.srtext::text, 'GEOCCS["'::text, ''::text), 1, "position"(replace(sp.srtext::text, 'GEOCCS["'::text, ''::text), '",'::text) - 1)
WHEN "position"(sp.srtext::text, 'COMPD_CS'::text) = 1 THEN "substring"(replace(sp.srtext::text, 'COMPD_CS["'::text, ''::text), 1, "position"(replace(sp.srtext::text, 'COMPD_CS["'::text, ''::text), '",'::text) - 1)
ELSE 'Non défini'::text
END AS sys_coord,
st_srid(s.the_geom_4326) AS "SRID",
st_astext(s.the_geom_4326) AS localisation_wkt,
st_x(st_centroid(s.the_geom_4326)) AS coord_x,
st_y(st_centroid(s.the_geom_4326)) AS coord_y,
s."precision" AS precision,
NULL::text AS nature_objet,
s.altitude_min AS alti_min,
s.altitude_max AS alti_max,
NULL::text AS pente,
NULL::text AS exposition,
NULL::text AS comm_geol,
NULL::text AS milieu,
s.observers AS observateurs,
s.date_min AS date_debut,
s.date_max AS date_fin,
NULL::text AS comm_context,
n2.mnemonique AS type_regroupement,
s.grp_method AS meth_regroupement,
NULL::text AS surface,
(s."additional_data"::json#>>'{strate_flore}') ||
ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)
AS strate_vegetation,
(s."additional_data"::json#>>'{heigth_herbace}')::text || (s."additional_data"::json#>>'{heigth_arbust}')::text
|| (s."additional_data"::json#>>'{heigth_arbore}')::text || (s."additional_data"::json#>>'{heigth_ssarbust}')::text
AS hauteur_strate,
(s."additional_data"::json#>>'{rcvmt_herbace}')::text || (s."additional_data"::json#>>'{rcvmt_arbust}')::text
|| (s."additional_data"::json#>>'{rcvmt_arbore}')::text || (s."additional_data"::json#>>'{rcvmt_ssarbust}')::text
AS recouvrement_strate,
s.cd_hab AS cdhab,
NULL::text AS cdhab_v,
NULL::text AS code_eur,
NULL::text AS code_eunis,
NULL::text AS code_cahab,
NULL::text AS code_cb,
NULL::text AS id_microhab,
n21.regne AS regne,
s.nom_cite AS nom_cite,
s.cd_nom as cd_nom,
(s."additional_data"::json#>>'{effectif_textuel}')::text ||
ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_braunblanq_abdom}')::int)::text
AS abondance,
NULL::text AS sociabilite,
n11.mnemonique AS sexe,
n7.mnemonique AS naturalite,
NULL::text AS comm_description,
n15.mnemonique AS statut_observation,
n12.mnemonique AS objet_denombrement,
n13.mnemonique AS type_denombrement,
s.count_min AS nombre_min,
s.count_max AS nombre_max,
n17.label_default AS statut_source,
NULL::text AS reference_biblio,
NULL::text AS page,
n8.mnemonique AS preuve_existence,
s.digital_proof AS preuve_numerique,
s.non_digital_proof AS preuve_non_numerique,
NULL::text AS nom_collection,
NULL::text AS ref_collection,
s.determiner AS determinateur,
NULL::text AS niv_val,
NULL::text AS niveau_diffusion,
n16.mnemonique AS floutage_dee,
NULL::text AS methode_observation,
CASE WHEN ts.name_source like 'MONITORING_%%'
THEN split_part(ts.name_source,'_',2)
ELSE 'RELEVE FLORE'::text
END AS protocole,
n6.mnemonique AS etat_biologique,
n5.mnemonique AS statut_biologique,
n10.mnemonique AS stade_vie,
n19.mnemonique AS methode_determination,
n3.mnemonique AS comportement,
s.additional_data - '{effectif_textuel,strate_flore,code_atlas,id_nomenclature_braunblanq_abdom,id_nomenclature_strate_flore}'::text[] AS donnees_additionnelles
FROM gn_synthese.synthese s
JOIN ds ON ds.id_dataset = s.id_dataset
JOIN af ON ds.id_acquisition_framework = af.id_acquisition_framework
JOIN geo ON s.id_synthese = geo.id_synthese
JOIN spatial_ref_sys sp ON st_srid(s.the_geom_4326) = sp.auth_srid
LEFT JOIN ref_habitats.habref h ON h.cd_hab = s.cd_hab
LEFT JOIN pr_occtax.t_releves_occtax occ ON occ.unique_id_sinp_grp = s.unique_id_sinp_grp
LEFT JOIN ref_nomenclatures.t_nomenclatures n1 ON s.id_nomenclature_geo_object_nature = n1.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n2 ON s.id_nomenclature_grp_typ = n2.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n3 ON s.id_nomenclature_behaviour = n3.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n4 ON s.id_nomenclature_obs_technique = n4.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n5 ON s.id_nomenclature_bio_status = n5.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n6 ON s.id_nomenclature_bio_condition = n6.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n7 ON s.id_nomenclature_naturalness = n7.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n8 ON s.id_nomenclature_exist_proof = n8.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n9 ON s.id_nomenclature_diffusion_level = n9.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n10 ON s.id_nomenclature_life_stage = n10.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n11 ON s.id_nomenclature_sex = n11.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n12 ON s.id_nomenclature_obj_count = n12.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n13 ON s.id_nomenclature_type_count = n13.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n14 ON s.id_nomenclature_sensitivity = n14.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n15 ON s.id_nomenclature_observation_status = n15.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n16 ON s.id_nomenclature_blurring = n16.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n17 ON s.id_nomenclature_source_status = n17.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n18 ON s.id_nomenclature_info_geo_type = n18.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n19 ON s.id_nomenclature_determination_method = n19.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n20 ON s.id_nomenclature_valid_status = n20.id_nomenclature
LEFT JOIN taxonomie.taxref n21 ON s.cd_nom = n21.cd_nom
JOIN gn_synthese.cor_area_synthese cas ON cas.id_synthese = s.id_synthese
-- ADD CEN_38
JOIN gn_synthese.t_sources ts ON ts.id_source = s.id_source
WHERE n21.regne::text = 'Plantae'::text
-- exclision CA ([SICEN] Données anciennes,[TEST] : migration sicen,[SICEN] Hors Etude)
AND ds.id_acquisition_framework NOT IN (53,65,66,1,2,6)
-- exclision JDD `Observations opportunistes du CEN Isère importé depuis Faune Isère`
AND ds.id_dataset NOT IN (185,377,236)
-- exclision CA (`Gestion de l'Espace Naturel Sensible du Méandre des Oves`, `Gestion de la Réserve Naturelle Nationale de l'Ile de la Platière`, `Gestion des Natura 2000 FR 8201749 « Milieux alluviaux et aquatiques de lÎle de la Platière » et FR 8212012 « Île de la Platière »`, `RNN Platière`
--AND (ts.id_module = 7 AND ds.id_acquisition_framework IN (7,38,39,42,44,45,46,48,55,57,58,60)
AND (
unaccent(observers) LIKE ANY (array['%%Conservatoire d%%Espaces Naturels Isere%%', '%%Conservatoire d%%Espaces Naturels d%%Isere%%', '%%CEN Isere%%', '%%Association des Amis de l%%Ile de la Platiere%%'])
OR observers NOT LIKE '%%(%%)%%'
)
AND unaccent(observers) NOT LIKE ALL(array[
'%%Sympetrum%%','%%Departement%%','%%Rhone%%','%%Oxalis%%',
'%%LPO%%','%%GRPLS%%','%%Parvi%%','%%CD Isere%%','%%Personnel%%','%%Independant%%'
'%%DELOCHE Denis%%'
])
AND observers NOT IN ('', 'Benoit Dodelin',', ')
-- statut_validation = 'Certain - très probable'
-- AND n20.cd_nomenclature = '1'
ORDER BY s.id_synthese;
'''
with con_gn.begin() as cnx:
cnx.execute(sql)

View File

@ -46,7 +46,7 @@ CREATE VIEW gn_exports.v_synthese_sinp_with_metadata_flora_for_gn2pg AS
WHEN "position"(sp.srtext::text, 'COMPD_CS'::text) = 1 THEN "substring"(replace(sp.srtext::text, 'COMPD_CS["'::text, ''::text), 1, "position"(replace(sp.srtext::text, 'COMPD_CS["'::text, ''::text), '",'::text) - 1)
ELSE 'Non défini'::text
END AS sys_coord,
st_srid(s.the_geom_4326) AS "SRID",
st_srid(s.the_geom_4326) AS srid,
st_astext(s.the_geom_4326) AS localisation_wkt,
st_x(st_centroid(s.the_geom_4326)) AS coord_x,
st_y(st_centroid(s.the_geom_4326)) AS coord_y,
@ -69,8 +69,7 @@ CREATE VIEW gn_exports.v_synthese_sinp_with_metadata_flora_for_gn2pg AS
THEN (s."additional_data"::json#>>'{strate_flore}')::text
WHEN s."additional_data"::json#>>'{id_nomenclature_strate_flore}' IS NOT NULL
THEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text
ELSE NULL::text
END AS strate_vegetation,
ELSE NULL::text END AS strate_vegetation,
CASE WHEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text = 'Strate herbacée'::text
THEN (s."additional_data"::json#>>'{heigth_herbace}')::text
WHEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text = 'Strate arbustive'::text
@ -79,8 +78,8 @@ CREATE VIEW gn_exports.v_synthese_sinp_with_metadata_flora_for_gn2pg AS
THEN (s."additional_data"::json#>>'{heigth_arbore}')::text
WHEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text = 'Strate sous-arbustive'::text
THEN (s."additional_data"::json#>>'{heigth_ssarbust}')::text
ELSE NULL::text
END AS hauteur_strate,
ELSE NULL::text END
AS hauteur_strate,
CASE WHEN s."additional_data"::json#>>'{id_nomenclature_strate_flore}' IS NOT NULL
THEN CASE WHEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text = 'Strate herbacée'::text
THEN (s."additional_data"::json#>>'{rcvmt_herbace}')::text
@ -91,8 +90,7 @@ CREATE VIEW gn_exports.v_synthese_sinp_with_metadata_flora_for_gn2pg AS
WHEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_strate_flore}')::int)::text = 'Strate sous-arbustive'::text
THEN (s."additional_data"::json#>>'{rcvmt_ssarbust}')::text
ELSE NULL::text END
ELSE NULL::text
END AS recouvrement_strate,
ELSE NULL::text END AS recouvrement_strate,
s.cd_hab AS cdhab,
NULL::text AS cdhab_v,
NULL::text AS code_eur,
@ -107,8 +105,7 @@ CREATE VIEW gn_exports.v_synthese_sinp_with_metadata_flora_for_gn2pg AS
THEN (s."additional_data"::json#>>'{effectif_textuel}')::text
WHEN s."additional_data"::json#>>'{id_nomenclature_braunblanq_abdom}' IS NOT NULL
THEN ref_nomenclatures.get_nomenclature_label((s."additional_data"::json#>>'{id_nomenclature_braunblanq_abdom}')::int)::text
ELSE NULL::text
END AS abondance,
ELSE NULL::text END AS abondance,
NULL::text AS sociabilite,
n11.mnemonique AS sexe,
n7.mnemonique AS naturalite,
@ -133,14 +130,13 @@ CREATE VIEW gn_exports.v_synthese_sinp_with_metadata_flora_for_gn2pg AS
NULL::text AS methode_observation,
CASE WHEN ts.name_source like 'MONITORING_%%'
THEN split_part(ts.name_source,'_',2)
ELSE 'RELEVE FLORE'::text
END AS protocole,
ELSE 'RELEVE FLORE'::text END AS protocole,
n6.mnemonique AS etat_biologique,
n5.mnemonique AS statut_biologique,
n10.mnemonique AS stade_vie,
n19.mnemonique AS methode_determination,
n3.mnemonique AS comportement,
s.additional_data - '{effectif_textuel,strate_flore,code_atlas,id_nomenclature_braunblanq_abdom,id_nomenclature_strate_flore}'::text[] AS donnees_additionnelles
s.additional_data - '{effectif_textuel,strate_flore,code_atlas,id_nomenclature_braunblanq_abdom,id_nomenclature_strate_flore}'::text[] AS additional_data
FROM gn_synthese.synthese s
JOIN ds ON ds.id_dataset = s.id_dataset
JOIN af ON ds.id_acquisition_framework = af.id_acquisition_framework
@ -151,21 +147,21 @@ CREATE VIEW gn_exports.v_synthese_sinp_with_metadata_flora_for_gn2pg AS
LEFT JOIN ref_nomenclatures.t_nomenclatures n1 ON s.id_nomenclature_geo_object_nature = n1.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n2 ON s.id_nomenclature_grp_typ = n2.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n3 ON s.id_nomenclature_behaviour = n3.id_nomenclature
-- LEFT JOIN ref_nomenclatures.t_nomenclatures n4 ON s.id_nomenclature_obs_technique = n4.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n4 ON s.id_nomenclature_obs_technique = n4.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n5 ON s.id_nomenclature_bio_status = n5.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n6 ON s.id_nomenclature_bio_condition = n6.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n7 ON s.id_nomenclature_naturalness = n7.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n8 ON s.id_nomenclature_exist_proof = n8.id_nomenclature
-- LEFT JOIN ref_nomenclatures.t_nomenclatures n9 ON s.id_nomenclature_diffusion_level = n9.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n9 ON s.id_nomenclature_diffusion_level = n9.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n10 ON s.id_nomenclature_life_stage = n10.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n11 ON s.id_nomenclature_sex = n11.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n12 ON s.id_nomenclature_obj_count = n12.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n13 ON s.id_nomenclature_type_count = n13.id_nomenclature
-- LEFT JOIN ref_nomenclatures.t_nomenclatures n14 ON s.id_nomenclature_sensitivity = n14.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n14 ON s.id_nomenclature_sensitivity = n14.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n15 ON s.id_nomenclature_observation_status = n15.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n16 ON s.id_nomenclature_blurring = n16.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n17 ON s.id_nomenclature_source_status = n17.id_nomenclature
-- LEFT JOIN ref_nomenclatures.t_nomenclatures n18 ON s.id_nomenclature_info_geo_type = n18.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n18 ON s.id_nomenclature_info_geo_type = n18.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n19 ON s.id_nomenclature_determination_method = n19.id_nomenclature
LEFT JOIN ref_nomenclatures.t_nomenclatures n20 ON s.id_nomenclature_valid_status = n20.id_nomenclature
LEFT JOIN taxonomie.taxref n21 ON s.cd_nom = n21.cd_nom

View File

@ -227,7 +227,6 @@ LEFT JOIN outflow USING (zh_uuid)
LEFT JOIN functions USING (zh_uuid)
LEFT JOIN especes USING (zh_uuid)
--group BY zh_uuid
WHERE code not like '38CENIS%'
order by code
;
'''

View File

@ -1,22 +0,0 @@
from pycen import con_gn,update_to_sql
from os import path
import geopandas as gpd
PATH = '/media/colas/SRV/FICHIERS'
FILE = 'OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/ETUDES/Zones humides/ACTUALISATION 2024/BE EPODE - LEZE/KIT INVENTAIRE38 MODIFIE/a envoyer prêt/GEOM/38CG0105_Epode2025 (à intégrer).gpkg'
tmp = (gpd.read_file(path.join(PATH, FILE))
.rename(columns={'CODE_ZH':'code'})
.rename_geometry('geom')
)
if tmp.crs.srs.lower() != 'epsg:4326':
tmp = tmp.to_crs(4326)
update_to_sql(
tmp[['code','geom']],
con_gn,
't_zh',
'pr_zh',
'code'
)

View File

@ -7,15 +7,11 @@ PATH = '/home/colas/Documents/9_PROJETS/1_ZH/2024/CARTO/Couches SIG'
FILE = 'ZH_zh DDT 2024_MJ_modif 2025.shp'
df = (gpd.read_file(path.join(PATH, FILE))
.rename(columns={'Code ZH':'code','date':'update_date'})
.rename_geometry('geom')
)
df.geom = df.geom.buffer(0).to_crs(4326)
.rename(columns={'Code ZH':'code','date':'update_date'}))
df['update_author'] = 24 # 24 = Mathieu Juton
update_to_sql(
df[['code','update_author', 'update_date','geom']],
df[['code','update_author', 'update_date']],
con_gn,
't_zh',
'pr_zh',

View File

@ -1,23 +0,0 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
import geopandas as gpd
from pycen import con_gn, update_to_sql
from os import path
PATH = '/media/colas/SRV/'
FILE = 'FICHIERS/OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/ETUDES/Zones humides/diag_zh_GAM/ZH_Séchilienne/zh_sapey.gpkg'
df = (gpd.read_file(path.join(PATH, FILE))
.rename(columns={'code_zh':'code'})
.rename_geometry('geom'))
if df.crs.srs.lower() != 'epsg:4326':
df = df.to_crs(4326)
update_to_sql(
df[['code','geom']],
con_gn,
't_zh',
'pr_zh',
'code'
)

View File

@ -1,138 +0,0 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from sqlalchemy import text
def check_missing_data(con):
sql = '''
WITH t1 AS (SELECT
--tro.*,too.*
tro.unique_id_sinp_grp
FROM pr_occtax.t_occurrences_occtax AS too
JOIN pr_occtax.t_releves_occtax tro USING (id_releve_occtax)
WHERE too.id_occurrence_occtax NOT IN (
SELECT cco.id_occurrence_occtax FROM pr_occtax.cor_counting_occtax cco
)
)
SELECT
s.id_synthese,
too.id_releve_occtax,
too.id_occurrence_occtax,
tro.id_digitiser,
s.date_min,
dataset_name,
dataset_shortname,
geom_local geom,
meta_device_entry,
tro.id_digitiser,
tro.date_min,
tro.date_max,
too.cd_nom,
too.nom_cite,
s.nom_cite nom_cite_synthese,
s.cd_nom cd_nom_synthese,
s.last_action
FROM pr_occtax.t_occurrences_occtax AS too
JOIN pr_occtax.t_releves_occtax tro USING (id_releve_occtax)
JOIN gn_meta.t_datasets USING (id_dataset)
LEFT JOIN pr_occtax.cor_counting_occtax cco USING (id_occurrence_occtax)
LEFT JOIN gn_synthese.synthese s on s.unique_id_sinp = cco.unique_id_sinp_occtax
WHERE tro.unique_id_sinp_grp IN (SELECT t1.unique_id_sinp_grp FROM t1)
AND (too.nom_cite != s.nom_cite OR s.nom_cite is null)
ORDER BY too.id_releve_occtax
;
'''
with con.begin() as cnx:
return cnx.execute(text(sql)).fetchall()
def check_error_synthese(con):
sql = '''
WITH t1 AS (SELECT
--tro.*,too.*
tro.unique_id_sinp_grp
FROM pr_occtax.t_occurrences_occtax AS too
JOIN pr_occtax.t_releves_occtax tro USING (id_releve_occtax)
)
SELECT
s.id_synthese,
too.id_releve_occtax,
too.id_occurrence_occtax,
tro.id_digitiser,
s.date_min,
dataset_name,
dataset_shortname,
geom_local geom,
meta_device_entry,
tro.id_digitiser,
tro.date_min,
tro.date_max,
too.cd_nom,
too.nom_cite,
s.nom_cite nom_cite_synthese,
s.cd_nom cd_nom_synthese,
s.last_action
FROM pr_occtax.t_occurrences_occtax AS too
JOIN pr_occtax.t_releves_occtax tro USING (id_releve_occtax)
JOIN gn_meta.t_datasets USING (id_dataset)
LEFT JOIN pr_occtax.cor_counting_occtax cco USING (id_occurrence_occtax)
LEFT JOIN gn_synthese.synthese s on s.unique_id_sinp = cco.unique_id_sinp_occtax
WHERE tro.unique_id_sinp_grp IN (SELECT t1.unique_id_sinp_grp FROM t1)
AND (too.nom_cite != s.nom_cite OR s.nom_cite is null)
ORDER BY too.id_releve_occtax
;
'''
with con.begin() as cnx:
return cnx.execute(text(sql)).fetchall()
def resolve_missing_data(con):
sql = '''
INSERT INTO pr_occtax.cor_counting_occtax (
id_occurrence_occtax,id_nomenclature_life_stage,id_nomenclature_sex,
id_nomenclature_obj_count,id_nomenclature_type_count,count_min,count_max,
additional_fields)
SELECT
-- t.operation_type,
-- t.operation_date,
-- CAST(t.table_content->>'id_counting_occtax' AS BIGINT) id_occurrence_occtax,
CAST(t.table_content->>'id_occurrence_occtax' AS BIGINT) AS id_occurrence_occtax,
CAST(t.table_content->>'id_nomenclature_life_stage' AS INT) AS id_nomenclature_life_stage,
CAST(t.table_content->>'id_nomenclature_sex' AS INT) AS id_nomenclature_sex,
CAST(t.table_content->>'id_nomenclature_obj_count' AS INT) AS id_nomenclature_obj_count,
CAST(t.table_content->>'id_nomenclature_type_count' AS INT) AS id_nomenclature_type_count,
CAST(t.table_content->>'count_min' AS INT) AS count_min,
CAST(t.table_content->>'count_max' AS INT) AS count_max,
CAST(t.table_content->>'additional_fields' AS JSON) AS additional_fields
FROM (
SELECT DISTINCT ON (d1.uuid_attached_row)
d1.uuid_attached_row, d1.*
FROM pr_occtax.t_occurrences_occtax o
INNER JOIN gn_commons.t_history_actions o1 ON o.unique_id_occurence_occtax = o1.uuid_attached_row
INNER JOIN gn_commons.t_history_actions d1 ON d1.table_content->>'id_occurrence_occtax' = o1.table_content->>'id_occurrence_occtax'
INNER JOIN gn_commons.t_history_actions d2 ON d2.uuid_attached_row = d1.uuid_attached_row
INNER JOIN gn_commons.t_history_actions o2 ON d2.table_content->>'id_occurrence_occtax' = o2.table_content->>'id_occurrence_occtax'
WHERE
NOT EXISTS (SELECT NULL FROM pr_occtax.cor_counting_occtax c WHERE o.id_occurrence_occtax = c.id_occurrence_occtax)
AND o1.id_table_location = 6 -- ID table t_occurrence_occtax
AND d1.id_table_location = 5 -- ID table cor_counting_occtax
AND d2.id_table_location = 5 -- ID table cor_counting_occtax
AND o2.id_table_location = 6 AND o2.operation_type = 'I' -- table t_occurrence_occtax
AND d2.table_content->>'id_occurrence_occtax' <> d1.table_content->>'id_occurrence_occtax'
AND d2.operation_date <> d1.operation_date
-- AND o1.table_content->>'id_releve_occtax' = '3522'
ORDER BY d1.uuid_attached_row, d1.operation_date DESC
) AS t
;'''
with con.begin() as cnx:
cnx.execute(text(sql))
def resolve_synthese_errors(con):
sql = '''
'''
if __name__ == '__main__':
from pycen import con_gn
check_missing_data(con_gn)
check_error_synthese(con_gn)

View File

@ -1,44 +0,0 @@
from pycen import con_gn
from sqlalchemy.sql import text
def update_additional_values(
con,table='cor_counting_occtax',shema='pr_occtax',
jsonb_column='additional_fields',keyfield='code_atlas'):
sql = text('''
UPDATE %(sch)s.%(tab)s
SET "%(jsonb_c)s" = JSONB_SET(
"%(jsonb_c)s",
'{%(keyf)s}'::text[],
TO_JSONB(
ref_nomenclatures.get_nomenclature_label(("%(jsonb_c)s"::json#>>'{%(keyf)s}')::int,
'default')
),
false)
WHERE ("%(jsonb_c)s"::json#>>'{%(keyf)s}')::text ~ '^[0-9\.]+$'
AND ("%(jsonb_c)s"::json#>>'{%(keyf)s}')::int in (
SELECT id_nomenclature FROM ref_nomenclatures.t_nomenclatures t
JOIN ref_nomenclatures.bib_nomenclatures_types b USING (id_type)
WHERE lower(b.mnemonique) = '%(keyf)s'
)
'''%{'keyf':keyfield,'tab':table,'sch':shema,'jsonb_c':jsonb_column})
# print(sql)
with con.begin() as cnx:
cnx.execute(sql)
def update_additional_key(
con,table='cor_counting_occtax',shema='pr_occtax',jsonb_column='additional_fields',
keyfield='Strate de végétation',new_keyfield='strate_flore'
):
sql = text('''
UPDATE %(sch)s.%(tab)s
SET "%(jsonb_c)s" = "%(jsonb_c)s" - '%(keyf)s' || jsonb_build_object(
'%(new_keyf)s', "%(jsonb_c)s"->'%(keyf)s'
)
WHERE "%(jsonb_c)s" ? '%(keyf)s';
'''%{'keyf':keyfield,'new_keyf':new_keyfield,'tab':table,'sch':shema,'jsonb_c':jsonb_column})
# print(sql)
with con.begin() as cnx:
cnx.execute(sql)