CEN38/Python_scripts
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init depot renew

Browse Source
This commit is contained in:
Colas
2026-09-22 11:58:57 +02:00
commit e9fa6a7309
205 changed files with 39721 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -0,0 +1 @@
settings.json
0_FONCIER/CADASTRE/agregg_cadastre.py
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from sqlalchemy.sql import text
from sqlalchemy.engine import URL
from sqlalchemy import create_engine
import pandas as pd
import geopandas as gpd
from pycen import con_cad
# pg_dump -h 91.134.194.221 -d cadastre -U cgeier --schema="38_202207" -s > 38_202207.sql
# psql -h 91.134.194.221 -U cgeier -d cadastre -a -f 38_202207.sql
lst_sch = con_cad.dialect.get_schema_names(con_cad)
lst_sch = ['07_202207','26_202207','42_202207']
for s in lst_sch:
lst_tab = con_cad.dialect.get_table_names(con_cad,s)
for t in lst_tab:
lst_col = con_cad.dialect.get_columns(con_cad,t,s)
lst_col = [x['name'] for x in lst_col ]
if 'geom' in lst_col:
df = gpd.read_postgis('SELECT * FROM "{sch}".{tab}'.format(sch=s,tab=t),con_cad)
if df.empty:
continue
df.to_postgis(t,con_cad,'xx_202207',if_exists='append',index=False)
else :
df = pd.read_sql_table(t,con_cad,s)
if df.empty:
continue
df.to_sql(t,con_cad,'xx_202207',if_exists='append',index=False)
print("{sch}.{tab}".format(sch=s,tab=t))
0_FONCIER/CADASTRE/transfert_cadastre.py
+11
View File
@@ -0,0 +1,11 @@
# RESTORE AU NOM DE LA BDD ORIGINE
# pg_dump -C -h localhost -U localuser dbname | psql -h remotehost -U remoteuser dbname
# pg_dump -C -h 172.17.0.2 -U postgres -d postgres --schema="07_202107" | psql -h 91.134.194.221 -U cgeier -d bd_cen
# RESTORE AU NOM DE LA BDD CIBLE
# pg_dump -C -h 172.17.0.2 -U postgres -d postgres --schema="*_202501" --format=custom | pg_restore -h 91.134.194.221 -U cgeier --dbname="cadastre"
# export PGPASSWORD="xxxxxx";pg_dump -C -h localhost -U cgeier -d cadastre --schema="*_202603" | psql -h 91.134.194.221 -U cgeier --dbname="bd_cen" -W
# export PGPASSWORD="xxxxxx";pg_dump -C -h 91.134.194.221 -U cgeier -d cadastre --schema="*_202603" --format=custom | pg_restore -h 91.134.194.221 -U cgeier --dbname="bd_cen"
0_FONCIER/CADASTRE/view_cadastre.py
+70
View File
@@ -0,0 +1,70 @@
from pycen import con_fon
view_typprop_x = '''
DROP VIEW IF EXISTS {sch}.{view};
CREATE OR REPLACE VIEW {sch}.{view}
AS
SELECT
p.par_id,
p.codcom,
p.ccopre,
p.ccosec,
p.dnupla,
p.dparpi,
p.dcntpa,
p.ccocomm,
p.ccoprem,
p.ccosecm,
p.dnuplam,
p.type,
p.geom,
typprop_lib,
typprop,
array_agg(dnuper) dnuper,
array_agg(ddenom) ddenom
FROM "{sch}".parcelles{_cen} p
LEFT JOIN "{sch}".d_typprop USING (typprop_id)
LEFT JOIN "{sch}".lots{_cen} USING (par_id)
LEFT JOIN "{sch}".cadastre{_cen} USING (lot_id)
LEFT JOIN "{sch}".cptprop{_cen} USING (dnupro)
LEFT JOIN "{sch}".r_prop_cptprop{_cen} USING (dnupro)
LEFT JOIN "{sch}".proprios{_cen} USING (dnuper)
GROUP BY 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
'''
grant = """
GRANT ALL ON TABLE {sch}.{view} TO grp_admin;
GRANT SELECT ON TABLE {sch}.{view} TO grp_consult;
"""
with con_fon.begin() as cnx:
cnx.execute(view_typprop_x.format(sch='cadastre', _cen= '', view='v_prop_parcelles'))
cnx.execute(view_typprop_x.format(sch='cadastre', _cen= '_cen', view='v_prop_parcelles_cen'))
cnx.execute(grant.format(sch='cadastre', view='v_prop_parcelles'))
cnx.execute(grant.format(sch='cadastre', view='v_prop_parcelles_cen'))
view_natcult_x = '''
DROP VIEW IF EXISTS {sch}.{view};
CREATE OR REPLACE VIEW {sch}.{view}
AS
SELECT
p.par_id,
p.geom,
array_agg(d_dsgrpf.natcult_ssgrp) natcult_ssgrp,
array_agg(d_cgrnum.natcult_grp) natcult_grp,
array_agg(d_cnatsp.natcult_natspe) natcult_natspe
FROM "{sch}".parcelles{_cen} p
LEFT JOIN "{sch}".lots{_cen} USING (par_id)
LEFT JOIN "{sch}".lots_natcult{_cen} USING (lot_id)
LEFT JOIN "{sch}".d_dsgrpf USING (dsgrpf)
LEFT JOIN "{sch}".d_cgrnum USING (cgrnum)
LEFT JOIN "{sch}".d_cnatsp USING (cnatsp)
GROUP BY par_id
'''
with con_fon.begin() as cnx:
cnx.execute(view_natcult_x.format(sch='cadastre', _cen= '', view='v_natcult'))
cnx.execute(view_natcult_x.format(sch='cadastre', _cen= '_cen', view='v_natcult_cen'))
cnx.execute(grant.format(sch='cadastre', view='v_natcult'))
cnx.execute(grant.format(sch='cadastre', view='v_natcult_cen'))
0_FONCIER/EXTRANET/create_siteTable.py
+239
View File
@@ -0,0 +1,239 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from pycen import con_fon
d_typinfosite = '''
CREATE TABLE IF NOT EXISTS sites.d_typinfosite (
typinfosite_id serial NOT NULL,
typinfosite_lib varchar(50) NOT NULL COLLATE pg_catalog."default",
CONSTRAINT d_typinfosite_pkey PRIMARY KEY (typinfosite_id)
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS sites.d_typinfosite OWNER to grp_sig;
GRANT SELECT ON TABLE sites.d_typinfosite TO cen_user;
GRANT DELETE, INSERT, UPDATE ON TABLE sites.d_typinfosite TO grp_foncier;
GRANT ALL ON TABLE sites.d_typinfosite TO grp_sig;
'''
ins_typinfosite = '''
INSERT INTO sites.d_typinfosite (typinfosite_lib) VALUES
('Essentiel'),
('Historique'),
('Patrimoine'),
('Objectifs'),
('Enjeux')
;
'''
r_sites_infos = '''
DROP TABLE IF EXISTS sites.r_sites_infos;
CREATE TABLE IF NOT EXISTS sites.r_sites_infos (
infosite_id serial NOT NULL,
site_id varchar(10) NOT NULL,
typinfosite_id integer NOT NULL,
site_info text,
maj_user text,
maj_date varchar(20),
CONSTRAINT r_sites_infos_pkey PRIMARY KEY (infosite_id),
CONSTRAINT r_sites_infos_site_id_fkey FOREIGN KEY (site_id)
REFERENCES sites.sites (site_id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION,
CONSTRAINT r_sites_infos_typinfosite_id_fkey FOREIGN KEY (typinfosite_id)
REFERENCES sites.d_typinfosite (typinfosite_id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS sites.r_sites_infos OWNER to grp_sig;
GRANT SELECT ON TABLE sites.r_sites_infos TO cen_user;
GRANT DELETE, INSERT, UPDATE ON TABLE sites.r_sites_infos TO grp_foncier;
GRANT ALL ON TABLE sites.r_sites_infos TO grp_sig;
CREATE INDEX IF NOT EXISTS fki_r_sites_infos_infosite_id_fkey
ON sites.r_sites_infos USING btree
(infosite_id ASC NULLS LAST)
TABLESPACE pg_default;
CREATE INDEX IF NOT EXISTS fki_r_sites_infos_site_id_fkey
ON sites.r_sites_infos USING btree
(site_id ASC NULLS LAST)
TABLESPACE pg_default;
CREATE INDEX IF NOT EXISTS fki_r_sites_infos_typinfosite_id_fkey
ON sites.r_sites_infos USING btree
(typinfosite_id ASC NULLS LAST)
TABLESPACE pg_default;
'''
with con_fon.begin() as cnx:
cnx.execute(d_typinfosite)
cnx.execute(ins_typinfosite)
cnx.execute(r_sites_infos)
secteurs = '''
CREATE TABLE IF NOT EXISTS sites.secteurs (
id_secteur serial NOT NULL,
site_id varchar(10) NOT NULL,
secteur_nom text NOT NULL COLLATE pg_catalog."default",
CONSTRAINT secteurs_pkey PRIMARY KEY (id_secteur),
CONSTRAINT secteurs_site_id_fkey FOREIGN KEY (site_id)
REFERENCES sites.sites (site_id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS sites.secteurs OWNER to grp_sig;
GRANT SELECT ON TABLE sites.secteurs TO cen_user;
GRANT DELETE, INSERT, UPDATE ON TABLE sites.secteurs TO grp_foncier;
GRANT ALL ON TABLE sites.secteurs TO grp_sig;
CREATE INDEX IF NOT EXISTS fki_secteurs_site_id_fkey
ON sites.secteurs USING btree
(site_id ASC NULLS LAST)
TABLESPACE pg_default;
'''
d_typpartenaire = '''
DROP TABLE IF EXISTS sites.d_typpartenaire;
CREATE TABLE IF NOT EXISTS sites.d_typpartenaire (
typpartenaire_id serial NOT NULL,
typpartenaire_lib text NOT NULL,
CONSTRAINT d_typpartenaire_pkey PRIMARY KEY (typpartenaire_id)
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS sites.d_typpartenaire OWNER to grp_sig;
GRANT SELECT ON TABLE sites.d_typpartenaire TO cen_user;
GRANT DELETE, INSERT, UPDATE ON TABLE sites.d_typpartenaire TO grp_foncier;
GRANT ALL ON TABLE sites.d_typpartenaire TO grp_sig;
'''
ins_typpartenaire = '''
INSERT INTO sites.d_typpartenaire (typpartenaire_lib) VALUES
('Financeurs'),
('Partenaires')
;
'''
r_sites_partenaires = '''
CREATE TABLE IF NOT EXISTS sites.r_sites_partenaires (
partenaire_id serial NOT NULL,
site_id varchar(10) NOT NULL,
typpartenaire_id integer NOT NULL,
maj_user text,
maj_date varchar(20),
CONSTRAINT r_sites_partenaires_pkey PRIMARY KEY (partenaire_id,site_id,typpartenaire_id),
CONSTRAINT r_sites_partenaires_site_id_fkey FOREIGN KEY (site_id)
REFERENCES sites.sites (site_id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION,
CONSTRAINT r_sites_partenaires_typpartenaire_id_fkey FOREIGN KEY (typpartenaire_id)
REFERENCES sites.d_typpartenaire (typpartenaire_id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS sites.r_sites_partenaires OWNER to grp_sig;
GRANT SELECT ON TABLE sites.r_sites_partenaires TO cen_user;
GRANT DELETE, INSERT, UPDATE ON TABLE sites.r_sites_partenaires TO grp_foncier;
GRANT ALL ON TABLE sites.r_sites_partenaires TO grp_sig;
CREATE INDEX IF NOT EXISTS fki_r_sites_partenaires_site_id_fkey
ON sites.r_sites_partenaires USING btree
(site_id ASC NULLS LAST)
TABLESPACE pg_default;
'''
d_typactnotsite = '''
CREATE TABLE IF NOT EXISTS sites.d_typactnotsite (
typactnotsite_id varchar(6) NOT NULL,
typactnotsite_lib varchar(10) NOT NULL,
CONSTRAINT d_typactnotsite_pkey PRIMARY KEY (typactnotsite_id)
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS sites.d_typactnotsite OWNER to grp_sig;
GRANT SELECT ON TABLE sites.d_typactnotsite TO cen_user;
GRANT DELETE, INSERT, UPDATE ON TABLE sites.d_typactnotsite TO grp_foncier;
GRANT ALL ON TABLE sites.d_typactnotsite TO grp_sig;
'''
actions_sites = '''
CREATE TABLE IF NOT EXISTS sites.actions_sites (
action_site_id serial NOT NULL,
action_site_date varchar(20) NOT NULL,
action_site_datemef varchar(20) NOT NULL,
typactnotsite_id varchar(6) NOT NULL,
action_site_lib text NOT NULL,
CONSTRAINT actions_sites_pkey PRIMARY KEY (action_site_id),
CONSTRAINT r_sites_partenaires_typactnotsite_id_fkey FOREIGN KEY (typactnotsite_id)
REFERENCES sites.d_typactnotsite (typactnotsite_id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS sites.actions_sites OWNER to grp_sig;
GRANT SELECT ON TABLE sites.actions_sites TO cen_user;
GRANT DELETE, INSERT, UPDATE ON TABLE sites.actions_sites TO grp_foncier;
GRANT ALL ON TABLE sites.actions_sites TO grp_sig;
'''
r_sites_actions = '''
CREATE TABLE IF NOT EXISTS sites.r_sites_actions (
action_site_id serial NOT NULL,
site_id varchar(10) NOT NULL,
maj_user text,
maj_date varchar(20),
actif varchar(1) NOT NULL,
CONSTRAINT r_sites_actions_pkey PRIMARY KEY (action_site_id,site_id),
CONSTRAINT r_sites_actions_site_id_fkey FOREIGN KEY (site_id)
REFERENCES sites.sites (site_id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION,
CONSTRAINT r_sites_actions_action_site_id_fkey FOREIGN KEY (action_site_id)
REFERENCES sites.actions_sites (action_site_id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS sites.r_sites_actions OWNER to grp_sig;
GRANT SELECT ON TABLE sites.r_sites_actions TO cen_user;
GRANT DELETE, INSERT, UPDATE ON TABLE sites.r_sites_actions TO grp_foncier;
GRANT ALL ON TABLE sites.r_sites_actions TO grp_sig;
'''
notes_sites = '''
CREATE TABLE IF NOT EXISTS sites.notes_sites (
note_site_id serial NOT NULL,
site_id varchar(10) NOT NULL,
note_site_date varchar(20) NOT NULL,
observateur_id integer NOT NULL,
typactnotsite_id integer NOT NULL,
note_site_lib text NOT NULL,
maj_user text,
maj_date varchar(20),
CONSTRAINT notes_sites_pkey PRIMARY KEY (note_site_id),
CONSTRAINT notes_sites_site_id_fkey FOREIGN KEY (site_id)
REFERENCES sites.sites (site_id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS sites.notes_sites OWNER to grp_sig;
GRANT SELECT ON TABLE sites.notes_sites TO cen_user;
GRANT DELETE, INSERT, UPDATE ON TABLE sites.notes_sites TO grp_foncier;
GRANT ALL ON TABLE sites.notes_sites TO grp_sig;
'''
with con_fon.begin() as cnx:
cnx.execute(d_typactnotsite)
cnx.execute(actions_sites)
cnx.execute(r_sites_actions)
0_FONCIER/EXTRANET/create_viewTravaux.py
+68
View File
@@ -0,0 +1,68 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from pycen import con_fon as con
from sqlalchemy import text
db = 'bd_cen'
host = '91.134.194.221'
port = '5432'
srv_name = 'fdw_bd-cen-38'
tab_name = 'cr_line_saisie_travaux'
sch_name = 'travaux'
view_name = 'travaux_ligne'
user = 'cen_admin'
pwd = '#CEN38@venir'
dict_cols = {
'gid': 'serial4',
'geom': 'public.geometry(linestring, 2154)',
'id_site': 'varchar',
'annee_trav': 'int4',
'id_gestion': 'int4',
'id_presta': 'int4',
'etat_trav': 'varchar',
'nom_presta': 'varchar',
'date_debut': 'date',
'duree_trav': 'float8',
'nb_etp': 'float8',
'nb_balle': 'int4',
'val_mat': 'int4',
'legende': 'varchar',
'rmq': 'varchar',
'prestation': 'int4',
'long_m': 'float8',
'crea_date': 'date',
'date_update': 'date',
'date_fin': 'date',
}
col_table = ','.join([' '.join([key,val]) for key, val in dict_cols.items()])
table = '''
DROP FOREIGN TABLE IF EXISTS {sch_name}.{tab_name} CASCADE;
CREATE FOREIGN TABLE IF NOT EXISTS {sch_name}.{tab_name} (
{columns}
)
SERVER "{fgn_server}"
OPTIONS (schema_name '{sch_name}', table_name '{tab_name}');
'''.format(
sch_name=sch_name, tab_name=tab_name, columns=col_table, fgn_server=srv_name
)
col_view = ','.join(dict_cols.keys())
view = '''
CREATE OR REPLACE VIEW {sch_name}.{v_name} AS
SELECT
{columns}
FROM {sch_name}.{tab_name};
ALTER TABLE {sch_name}.{v_name} OWNER TO cen_admin;
GRANT ALL ON TABLE {sch_name}.{v_name} TO cen_admin;
GRANT SELECT ON TABLE {sch_name}.{v_name} TO grp_sig;
GRANT SELECT ON TABLE {sch_name}.{v_name} TO cen_user;
'''.format(
sch_name=sch_name, tab_name=tab_name, v_name=view_name, columns=col_view
)
with con.begin() as cnx:
cnx.execute(text(table))
cnx.execute(text(view))
0_FONCIER/TdB_FEDE/__init__.py
+31
View File
@@ -0,0 +1,31 @@
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
from os import getlogin
import pandas as pd
import geopandas as gpd
from sqlalchemy.engine import URL
from sqlalchemy import create_engine
from pycen import con_bdcen as con, con_fon
annee = 2023
path_source = '/media/{log}/SRV/FICHIERS/OUTILS/BASES DE DONNEES/BILAN_FEDE_CEN/{an}/Docs_SIG_joints{an}/Docs_SIG_joints{an}/'.format(log=getlogin(),an=annee)
dico_data = 'Dico_DATA_sites_CEN_v%i_v1.xlsx' % annee
sheet_par = 'parcelles_cen_XX_%i' % annee
sheet_sit = 'sites_cen_XX_%i' % annee
###### Récupération des données conservatoires
def get_sites_cen():
sqlsit = '''SELECT * FROM %s.%s WHERE date_fin is NULL OR date_fin >= '%i-01-01' ;'''%('sites','c_sites_zonages',annee)
dfsite = gpd.read_postgis(sql=sqlsit,geom_col='geom',con=con)
# Isolation de tous les sites CEN (gestion/assist/mili)
return dfsite[
~dfsite.code_site.duplicated()|\
(dfsite.code_site.duplicated() & dfsite.type_zonage == 'ZI')
]
def get_parcelles_cen():
return
0_FONCIER/TdB_FEDE/parcelles_TdB_fede .py
+732
View File
@@ -0,0 +1,732 @@
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
# library pandas geopandas sqlalchemy psycopg2-binary openpyxl
from calendar import c
from os import getlogin
import pandas as pd
import geopandas as gpd
from sqlalchemy.sql import text
from sqlalchemy.engine import URL
from sqlalchemy import create_engine
from pycen import con_bdcen as con, con_fon
from functools import reduce
annee = 2022
path_source = '/media/{login}/SRV/FICHIERS/OUTILS/BASES DE DONNEES/BILAN_FEDE_CEN/{an}/Docs_SIG_joints{an}/Docs_SIG_joints{an}/'.format(login=getlogin(),an=annee)
dico_data = 'Dico_DATA_sites_CEN_v%i_v1.xlsx' % annee
sheet_par = 'parcelles_cen_XX_%i' % annee
sheet_sit = 'sites_cen_XX_%i' % annee
lv_cen = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='A:D',nrows=25)
lv_typmaitr = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='F:K',nrows=23)
lv_typmilie = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='M:N',nrows=16)
lv_echelnum = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='P:Q',nrows=8)
lv_typhab = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=13, usecols='P:Q',nrows=5)
lv_sourcegeom = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='S:U',nrows=12)
lv_lienres = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=20, usecols='M:N',nrows=4)
lv_typprop = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=45, usecols='A:H',nrows=12)
lv_codgeol = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=36, usecols='M:T',nrows=19)
lst_cols_parc = pd.read_excel(path_source+dico_data,sheet_name=sheet_par,header=0, usecols='F',nrows=26)\
['nom du champ'].tolist()
user = 'postgres'
pwd = 'postgres'
adr = '172.17.0.2'
base = 'postgres'
url = URL.create('postgresql+psycopg2',
username=user,
password=pwd,
host=adr,
database=base,
)
con_vm = create_engine(url)
fadd = '/home/colas/Documents/tmp/FONCIER_FEDE/add_parc.gpkg'
add = gpd.read_file(fadd)
sql0 = '''SELECT
t1.geo_parcelle, substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) par_id,
t1.annee, t1.update_dat, t4.ccopre, t4.ccosec, t4.dcntpa, t1.geom, STRING_AGG(t3.gtoper::text,',') gtoper, STRING_AGG(t3.ccogrm::text,',') ccogrm,
STRING_AGG(CONCAT(t3.gtoper::text||COALESCE('_'||t3.ccogrm::text,'')),',') type_prop, STRING_AGG(t3.ddenom::text,',') ddenom
FROM "{sch}"."{t1}" t1
LEFT JOIN ("{sch}".parcelle_info t2
LEFT JOIN "{sch}".proprietaire t3 USING (comptecommunal))
USING (geo_parcelle)
LEFT JOIN "{sch}".parcelle t4 ON t1.geo_parcelle = t4.parcelle
'''.format(
sch='38_202207',
t1='v_geo_parcelle')
sql1 = '''WHERE substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) IN ('%s')
GROUP BY 1,2,3,4,5,6,7,8'''%"','".join(add.par_id)
sql = sql0 + sql1
parcx = gpd.read_postgis(text(sql),con=con_vm)
cols_parcx = parcx.columns
for col in ['ccopre','ccosec','type_prop','ddenom'] :
parcx[col] = parcx[col].str.strip()
parcx[col] = parcx[col].replace('',None)
parcx.loc[parcx.ccopre.isna(),'ccopre'] = parcx.loc[parcx.ccopre.isna(),'par_id'].str[5:8]
parcx.loc[parcx.ccosec.isna(),'ccosec'] = parcx.loc[parcx.ccosec.isna(),'par_id'].str[8:10]
parcx.loc[parcx.ccosec.str.len()==1,'ccosec'] = '0'+parcx.loc[parcx.ccosec.str.len()==1,'ccosec']
parcx.loc[parcx.dcntpa.isna(),'dcntpa'] = parcx.loc[parcx.dcntpa.isna(),'geom'].area.round().astype(int)
parcx.loc[parcx.gtoper.isna(),['gtoper','ccogrm','type_prop']] = ['2','0','2_0']
parcx.loc[parcx.par_id=='380740000B0705','ddenom'] = "CONSERVATOIRE D'ESPACES NATURELS ISERE"
parcx.dcntpa = parcx.dcntpa.astype(int)
###### Récupération des parcelles RN île de la Platière
f_plaRN = '/home/colas/Documents/tmp/FONCIER_FEDE/PLATIERE/Maitrise_Foncier_RN.shp'
f_plaENS = '/home/colas/Documents/tmp/FONCIER_FEDE/PLATIERE/Maitrise_Foncier_ENS.shp'
RNNP_rn = gpd.read_file(f_plaRN)
RNNP_ens = gpd.read_file(f_plaENS)
RNNP_rn.rename_geometry('geom',inplace=True)
RNNP_ens.rename_geometry('geom',inplace=True)
RNNP_rn.rename(columns={'Date_sign':'annee_acq','surf_acqui':'surface_ac','surf_parce':'superficie'},inplace=True)
RNNP_ens.rename(columns={'code_commu':'code_comm','année_acq':'annee_acq','année':'anne'},inplace=True)
RNNP_rn = RNNP_rn[RNNP_rn.code_comm.str.len()==5]
RNNP_ens = RNNP_ens[RNNP_ens.code_comm.str.len()==5]
RNNP_rn['section'] = RNNP_rn.parcelle.str[:2]
RNNP_ens['section'] = RNNP_ens.parcelle.str[:2]
RNNP_rn['num_parc'] = RNNP_rn.parcelle.str[2:]
RNNP_ens['num_parc'] = RNNP_ens.parcelle.str[2:]
RNNP_rn['num_parc'] = RNNP_rn['num_parc'].str.strip()
RNNP_ens['num_parc'] = RNNP_ens['num_parc'].str.strip()
RNNP_rn.loc[RNNP_rn.num_parc.str.len()==1,'num_parc'] = '000' + RNNP_rn.loc[RNNP_rn.num_parc.str.len()==1,'num_parc']
RNNP_rn.loc[RNNP_rn.num_parc.str.len()==2,'num_parc'] = '00' + RNNP_rn.loc[RNNP_rn.num_parc.str.len()==2,'num_parc']
RNNP_rn.loc[RNNP_rn.num_parc.str.len()==3,'num_parc'] = '0' + RNNP_rn.loc[RNNP_rn.num_parc.str.len()==3,'num_parc']
RNNP_ens.loc[RNNP_ens.num_parc.str.len()==1,'num_parc'] = '000' + RNNP_ens.loc[RNNP_ens.num_parc.str.len()==1,'num_parc']
RNNP_ens.loc[RNNP_ens.num_parc.str.len()==2,'num_parc'] = '00' + RNNP_ens.loc[RNNP_ens.num_parc.str.len()==2,'num_parc']
RNNP_ens.loc[RNNP_ens.num_parc.str.len()==3,'num_parc'] = '0' + RNNP_ens.loc[RNNP_ens.num_parc.str.len()==3,'num_parc']
RNNP_rn.loc[RNNP_rn.code_comm.str.startswith('39'),'code_comm'] = '39' + RNNP_rn.loc[RNNP_rn.code_comm.str.startswith('39'),'code_comm'].str[2:]
RNNP_ens.loc[RNNP_ens.code_comm.str.startswith('39'),'code_comm'] = '39' + RNNP_ens.loc[RNNP_ens.code_comm.str.startswith('39'),'code_comm'].str[2:]
RNNP_rn['par_id'] = RNNP_rn.code_comm+'000'+RNNP_rn.section+RNNP_rn.num_parc
RNNP_ens['par_id'] = RNNP_ens.code_comm+'000'+RNNP_ens.section+RNNP_ens.num_parc
lst_par_id_rnnp = [*RNNP_rn.par_id,*RNNP_ens.par_id]
RNNP = pd.concat([RNNP_rn,RNNP_ens])
RNNP['surf_parc_maitrise_m2'] = RNNP.surface_ac
RNNP['pour_part'] = 0
RNNP.loc[(~RNNP.proportion.isna())&(RNNP.proportion!=100),'pour_part'] = 1 # Récup après, cf. dfgere
frnnp_tofede = '/home/colas/Documents/tmp/FONCIER_FEDE/PLATIERE/parcelles_RNNP_tofede.gpkg'
rnnp_tofede = gpd.read_file(frnnp_tofede,layer='parcelles_RNNP_tofede')
rnnp_tofede.rename_geometry('geom', inplace=True)
rnnp_tofede['id_site_cen_parc'] = 0
rnnp_tofede['id_site_fcen_parc'] = None # OK
rnnp_tofede['insee_dep'] = rnnp_tofede.par_id.str[:2]
rnnp_tofede['insee_com'] = rnnp_tofede.par_id.str[2:5]
rnnp_tofede['num_section'] = rnnp_tofede.ccosec
rnnp_tofede['num_parcelle'] = rnnp_tofede.par_id
# rnnp_tofede['code_mfu1'] = None # Récup après, cf. dfgere
# rnnp_tofede['code_mfu2'] = None # Récup après, cf. dfgere
# rnnp_tofede['type_prop'] = None # Récup après, cf. parcb
# rnnp_tofede['mesure_compens'] = None # Récup après, cf. dfgere
rnnp_tofede['surf_ore_m2'] = 0
rnnp_tofede['date_debut_ore'] = None
rnnp_tofede['date_fin_ore'] = None
rnnp_tofede['doc_foncier'] = 0
rnnp_tofede['source_doc_foncier'] = 0
rnnp_tofede['parc_gestion_rnx'] = 0
rnnp_tofede['parc_gestion_rnx'] = 1
rnnp_tofede['surf_parc_maitrise_m2'] = rnnp_tofede.dcntpa
rnnp_tofede['source_geom_parc_nature'] = 2
rnnp_tofede['source_geom_parc_annee'] = None # Récup après, cf. parcb
rnnp_tofede['echelle_num_parc'] = 0
rnnp_tofede['source_surf_parc'] = 1 # OK
rnnp_tofede['date_maj_parcelle'] = None # Récup après, cf. parcb
rnnp_tofede['bnd'] = None # Récup après, cf. dfgere
rnnp_tofede['pour_part'] = None # Récup après, cf. dfgere
rnnp_tofede['domaine_public'] = 0
rnnp_tofede['id_proprietaire'] = None # Récup après, cf. parcb
# RNNP_ens[(~RNNP_ens.num_parcel.isna())&(RNNP_ens.num_parcel!=RNNP_ens.parcelle)]
###### Récupération des données conservatoires
sqlsit = '''SELECT * FROM %s.%s WHERE date_fin is NULL OR date_fin >= '%i-01-01' ;'''%('sites','c_sites_zonages',annee)
dfsite = gpd.read_postgis(sql=sqlsit,geom_col='geom',con=con)
# Isolation de tous les sites CEN (gestion/assist/mili)
dfsite[
~dfsite.code_site.duplicated()|\
(dfsite.code_site.duplicated() & dfsite.type_zonage == 'ZI')
]
rngl = dfsite[(dfsite.code_site=='RNGL')&(dfsite.type_zonage=='ZI')]
sqlfon = 'SELECT * FROM %s.%s'%('foncier','c_animation_fonciere_sites')
dffonc = gpd.read_postgis(sql=sqlfon,geom_col='geom',con=con)
##### Données identifiées pour les remontées fédé
ffede = '/home/colas/Documents/tmp/FONCIER_FEDE/extract_parc_for_fede.gpkg'
tofede = gpd.read_file(ffede)
tofede.rename_geometry('geom', inplace=True)
cols_tofede = tofede.columns
cols_join = [*cols_parcx[cols_parcx.isin(cols_tofede)]]
tofede = pd.concat([tofede,parcx[cols_join]],ignore_index=True)
tofede.reset_index(drop=True,inplace=True)
if 'type_prop' in tofede.columns:
tofede.drop(columns=['type_prop'], inplace=True)
# tofede['id_site_cen_parc'] = 0
tofede['id_site_fcen_parc'] = None # OK
tofede['insee_dep'] = tofede.par_id.str[:2]
tofede['insee_com'] = tofede.par_id.str[2:5]
tofede['num_section'] = tofede.ccosec
tofede['num_parcelle'] = tofede.par_id
# tofede['code_mfu1'] = None # Récup après, cf. dfgere
# tofede['code_mfu2'] = None # Récup après, cf. dfgere
# tofede['type_prop'] = None # Récup après, cf. parcb
# tofede['mesure_compens'] = None # Récup après, cf. dfgere
tofede['surf_ore_m2'] = 0
tofede['date_debut_ore'] = None
tofede['date_fin_ore'] = None
tofede['doc_foncier'] = 0
tofede['source_doc_foncier'] = 0
tofede['parc_gestion_rnx'] = 0
tofede.loc[tofede.geom.intersects(rngl.unary_union),'parc_gestion_rnx'] = 1
tofede['surf_parc_maitrise_m2'] = tofede.dcntpa
tofede['source_geom_parc_nature'] = 2
# tofede['source_geom_parc_annee'] = None # Récup après, cf. parcc
tofede['echelle_num_parc'] = 0
tofede['source_surf_parc'] = 1 # OK
# tofede['date_maj_parcelle'] = None # Récup après, cf. parcc
# tofede['bnd'] = None # Récup après, cf. dfgere
# tofede['pour_part'] = None # Récup après, cf. dfgere
tofede['domaine_public'] = 0
# tofede['id_proprietaire'] = None # Récup après, cf. parcb
# c_fede = tofede[['par_id','geom']].copy()
# c_fede.geom = tofede.representative_point()
# pb_parcel = dffonc[~dffonc.intersects(c_fede.unary_union)]
# c_fede.to_file('/home/colas/Documents/tmp/FONCIER_FEDE/centroid_parcelle_for_fede.gpkg', driver="GPKG")
# pb_parcel.to_file('/home/colas/Documents/tmp/FONCIER_FEDE/polys_NoIntersect_parcelles.gpkg', driver="GPKG")
###### Récupération des données conservatoires
# sqlsit = 'SELECT * FROM %s.%s'%('sites','c_sites_zonages')
# dfsite = gpd.read_postgis(sql=sqlsit,geom_col='geom',con=con)
sqlfon = 'SELECT * FROM %s.%s'%('foncier','c_animation_fonciere_sites')
dffonc = gpd.read_postgis(sql=sqlfon,geom_col='geom',con=con)
# dffonc.drop(columns=[], inplace=True)
#### 1 geom devient 2
# id 7037
dffonc.loc[dffonc.id==7037, ['code_dep','code_com','section','numero','geom']] = tofede.loc[tofede.par_id=='381000000A2233',['insee_dep','insee_com','num_section','dnupla','geom']].values[0]
add_p = gpd.GeoDataFrame(
[tofede.loc[tofede.par_id=='381000000A2234',['insee_dep','insee_com','num_section','dnupla','geom']].values[0]],
columns=['code_dep','code_com','section','numero','geom']
)
oth_cols = dffonc.columns[~dffonc.columns.isin(add_p.columns)].drop('id')
add_p[[*oth_cols]] = dffonc.loc[dffonc.id==7037, [*oth_cols]].values[0]
add_p['id'] = 0
add_p.set_geometry('geom', crs=2154, inplace=True)
dffonc = pd.concat([dffonc,add_p])
# id 1932
dffonc.loc[dffonc.id==1932, ['code_dep','code_com','section','numero','geom']] = tofede.loc[tofede.par_id=='38140000AM0190',['insee_dep','insee_com','num_section','dnupla','geom']].values[0]
add_p = gpd.GeoDataFrame(
[tofede.loc[tofede.par_id=='38140000AM0189',['insee_dep','insee_com','num_section','dnupla','geom']].values[0]],
columns=['code_dep','code_com','section','numero','geom']
)
oth_cols = dffonc.columns[~dffonc.columns.isin(add_p.columns)].drop('id')
add_p[[*oth_cols]] = dffonc.loc[dffonc.id==1932, [*oth_cols]].values[0]
add_p['id'] = 0
add_p.set_geometry('geom', crs=2154, inplace=True)
dffonc = pd.concat([dffonc,add_p])
####
dropline = dffonc[dffonc.id.isin([7144,7106,7107,7108,7109,7110,6666])].index.values # drop geometrie dupliquée
dffonc.drop([*dropline], inplace=True)
dffonc.loc[dffonc.numero.str.len()==1,'numero'] = '000'+dffonc.loc[dffonc.numero.str.len()==1,'numero']
dffonc.loc[dffonc.numero.str.len()==2,'numero'] = '00'+dffonc.loc[dffonc.numero.str.len()==2,'numero']
dffonc.loc[dffonc.numero.str.len()==3,'numero'] = '0'+dffonc.loc[dffonc.numero.str.len()==3,'numero']
dffonc.loc[dffonc.section.str.len()==1,'section'] = '0'+dffonc.loc[dffonc.section.str.len()==1,'section']
dffonc.loc[dffonc.id == 7101,['code_dep','code_com','section','numero']] = ['38','037','0D','0130']
dffonc.loc[dffonc.id == 7102,['code_dep','code_com','section','numero']] = ['38','037','0D','0131']
dffonc.loc[dffonc.id == 7103,['code_dep','code_com','section','numero']] = ['38','037','0D','0132']
dffonc.loc[dffonc.id == 7104,['code_dep','code_com','section','numero']] = ['38','037','0D','0133']
dffonc.loc[dffonc.id == 7105,['code_dep','code_com','section','numero']] = ['38','037','0D','0134']
dffonc.loc[dffonc.id.isin([
61,62,70,69,68,72,67,66,63,64,60,57,56,59,6,7,8,9,65,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,
26,27,28,29,30,58,31,32,33,34,35,36,37,38,39,40,41,55,54,42,43,71,44,45,46,47,48,49,50,51,52,53,4,5,
1350,1349,1345,1346,1347,1352,1353,1348,1354,1351, # ccopre == 121
2433,2447,2449,2432,2448 # ccopre == 274
]),'code_com'] = '479'
dffonc.loc[dffonc.id.isin([
2604,2605,2535,2599,2603,2602,2601,2639,2534,2536,2537,2532,2538,2540,2539,2635,2641,2533,2563,2562,
2561,2556,2560,2555,2597,2607,2614,2591,2622,2623,2620,2621,2619,2593,2592,2550,2547,2544,2551,2553,
2552,2545,2541,2546,2548,2549,2543,2583,2566,2596,2608,2567,2559,2584,2568,2615,2629,2618,2631,2571,
2609,2572,2590,2632,2569,2570,2595,2558,2636,2633,2606,2574,2573,2564,2579,2580,2577,2578,2565,2557,
2576,2581,2613,2598,2626,2627,2625,2611,2624,2642,2554,2628,2582,2585,2610,2542,2594,2589,2575,2616,
2612,2600,2617,2630,2587,2588,2586,2638,2637,2640,2634
]),'code_com'] = '292'
dffonc.loc[dffonc.id.isin([7114,7115,7116,7117,7118,7119,7126,7127,7128,7129,7130,7131,7132,7133,7134,7135]),'code_com'] = '064'
dffonc.loc[dffonc.id.isin([7120,7121,7122,7123,7124,7125]),'code_com'] = '401'
dffonc.loc[dffonc.id.isin([7136,7137,7138,7139]),['code_com','section']] = ['341','0B']
dffonc.loc[dffonc.id.isin([7142,7143]),'code_com'] = '052'
dffonc.loc[dffonc.id.isin([1010288,1041363]),'code_com'] = '517'
dffonc.loc[dffonc.id.isin([1182503,1182640]),'code_com'] = '147'
dffonc.loc[dffonc.id.isin([1243657]),'code_com'] = '448'
dffonc.loc[dffonc.id.isin([1629034,1629035]),'code_com'] = '257'
dffonc.loc[dffonc.code_dep.isna(),'code_dep'] = '38'
#####
dffonc.loc[dffonc.id==113, 'numero'] = '0389'
dffonc.drop(dffonc[dffonc.id==114].index[0],inplace=True)
dffonc.loc[dffonc.id==151, 'numero'] = '0369'
dffonc.loc[dffonc.id==591, 'numero'] = '0705' # numero == '0542' est un ancien nom
dffonc.sort_values(['code_dep','code_com','section','numero'],inplace=True)
dffonc['par_id'] = dffonc.code_dep + dffonc.code_com + '___' + dffonc.section + dffonc.numero
dffonc['parid_start'] = dffonc.code_dep + dffonc.code_com
dffonc['parid_end'] = dffonc.section + dffonc.numero
dffonc.loc[dffonc.id.isin([1634,1638,1606,1607,1608]),'par_id'] = dffonc.loc[dffonc.id.isin([1634,1638,1606,1607,1608]),'par_id'].replace('___','227',regex=True)
dffonc.loc[dffonc.id.isin([1495,1673,1675]),'par_id'] = dffonc.loc[dffonc.id.isin([1495,1673,1675]),'par_id'].replace('___','000',regex=True)
# dffonc.loc[dffonc.id.isin([1495,1675]),'par_id']
# Jointure by parid_start & parid_end
dfpar_id = tofede[['par_id']].copy()
dfpar_id['parid_start'] = dfpar_id.par_id.str[:5]
dfpar_id['parid_end'] = dfpar_id.par_id.str[-6:]
lst_parid_ok = [1495,1634,1638,1673,1675,1606,1607,1608] # liste parcelle par_id reconstitué entier
parid_ok = dffonc.loc[dffonc.id.isin(lst_parid_ok)]
parid_join = pd.merge(
dffonc.loc[~dffonc.id.isin(lst_parid_ok)].drop(columns=['par_id']),
dfpar_id.loc[~dfpar_id.par_id.isin(parid_ok.par_id)],
on = ['parid_start','parid_end'],
how='left'
)
parid_join.geom = parid_join.buffer(0)
parid_ok = pd.concat([ parid_ok, parid_join[~parid_join.par_id.isna()] ])
parid_ok.drop(columns=['parid_start','parid_end'], inplace=True)
parid_na = parid_join[parid_join.par_id.isna()]
# Jointure by centroid
dfpar_id = tofede.loc[~tofede.par_id.isin(parid_ok.par_id),['par_id','geom']].copy()
dfpar_id.geom = dfpar_id.representative_point()
parid_join = gpd.sjoin(parid_na.drop(columns=['par_id']),dfpar_id,op='contains', how='left').drop(columns=['index_right'])
parid_join_ok = parid_join[ [str(x).startswith(y) and z in str(x) for x,y,z in zip(parid_join['par_id'], parid_join['parid_start'],parid_join['section'])] ].copy()
# parid_join[ [str(x).endswith(z) for x,y,z in zip(parid_join['par_id'], parid_join['parid_start'],parid_join['parid_end'])] ]
parid_join_ok.drop(columns=['parid_start','parid_end'], inplace=True)
# Couches annimation foncière finale
parid_OK = pd.concat([parid_ok,parid_join_ok])
parid_NA = dffonc[~dffonc.id.isin(parid_OK.id.unique())]
# Récupération des info BND
sqlmod = 'SELECT * FROM %s'%('animation_fonciere_pour_fcen')
dfmodf = gpd.read_postgis(sql=sqlmod,geom_col='geom',con=con)
del dfmodf['id_0']
dffon_OK = pd.merge(
parid_OK,
dfmodf[['id','Parc_MCE','docs_off_num','Nbr_lots_BND','partiel_conv_ou_acq']],
on='id',how='left')
#
lst_prop = ['Acquis par le CEN ou CREN','ACQUIS&CONVENTION', 'PARTIELLEMENT MAITRISE']
lst_conv = ["CONVENTION D'USAGE",'CONVENTION']
lst_gere = ['BAIL EMPHYTEOTIQUE', *lst_conv, *lst_prop]
dfgere = dffon_OK[
# (
# (dffon_OK.geom.intersects(dfsite[(dfsite.type_zonage=='ZI')|(dfsite.type_site=='MC')].unary_union))|
# (~dffon_OK.Parc_MCE.isna())|(~dffon_OK.Nbr_lots_BND.isna())|
# (~dffon_OK.partiel_conv_ou_acq.isna())|(~dffon_OK.docs_off_num.isna())
# )
# &
(dffon_OK.maitrise.isin(lst_gere))
].copy()
dfautr = dffon_OK[~dffon_OK.par_id.isin(dfgere.par_id)].copy()
# id_site_cen_parc
dfgere['id_site_cen_parc'] = dfgere.id
# code_mfu1 & code_mfu2
dfgere['code_mfu1'] = None
dfgere['code_mfu2'] = None
dfgere.loc[dfgere.maitrise.isin(lst_prop),'code_mfu1'] = 'P'
dfgere.loc[dfgere.maitrise.isin(lst_conv),'code_mfu1'] = 'C'
dfgere.loc[dfgere.maitrise == 'BAIL EMPHYTEOTIQUE','code_mfu1'] = 'L'
dfgere.loc[dfgere.maitrise.isin(lst_prop),'code_mfu2'] = 'P1'
dfgere.loc[dfgere.maitrise.isin(lst_conv),'code_mfu2'] = 'C7'
dfgere.loc[dfgere.maitrise == 'BAIL EMPHYTEOTIQUE','code_mfu2'] = 'L1'
# BND
dfgere['bnd'] = 0
dfgere.loc[dfgere.indivision=='BND','bnd'] = 1
tmp = pd.DataFrame(columns=dfgere.columns)
for i,row in dfgere.loc[~dfgere.Nbr_lots_BND.isna()].iterrows():
# if i==0 : tmp = row.to_list()
tmp = tmp.append([row]*(int(row.Nbr_lots_BND)-1))
dfgere = pd.concat([dfgere,tmp]).sort_index()
# MC
dfgere['mesure_compens'] = 0
dfgere.loc[dfgere.Parc_MCE=='OUI','mesure_compens'] = 1
# pour_part
dfgere['pour_part'] = 0
dfgere.loc[dfgere.maitrise=='PARTIELLEMENT MAITRISE','pour_part'] = 1
# Parcelles millitaire
dic = {
'ccosec':'section',
}
dfmili = dfsite[dfsite.type_site=='MILI'].copy()
sqlmil = "SELECT * FROM %s.%s WHERE ST_Contains('SRID=2154;%s',geom)" % ('cadastre', 'parcelles_38',dfmili.buffer(10).unary_union)
miliparc = gpd.read_postgis(sqlmil,con=con_fon)
miliparc['code_dep'] = miliparc.codcom.str[:2]
miliparc['code_com'] = miliparc.codcom.str[2:]
dfmili = miliparc[['par_id','geom','ccosec','code_dep','code_com']].copy()
dfmili.rename(columns=dic, inplace=True)
dfmili['id_site_cen_parc'] = 0
dfmili['code_mfu1'] = 'C'
dfmili['code_mfu2'] = 'C12'
dfmili['bnd'] = 0
dfmili['mesure_compens'] = 0
dfmili['pour_part'] = 0
dfgere = pd.concat([dfgere,dfmili])
# from pycen import update_to_sql
# tmpdffonc = dffonc[['id','nom_propri']]
# tmpdffonc["nom_propri"] = tmpdffonc["nom_propri"].str.strip()
# tmpdffonc.loc[tmpdffonc.nom_propri== '',"nom_propri"] = None
# update_to_sql(df=tmpdffonc, con=con, table_name='c_animation_fonciere_sites', schema_name='foncier', key_name='id', geom_col=None)
#####################
##### Parcelles #####2017380474000AN
# Ne considère pas Platière, ni le Grand Lemps
#####################
# Récupération des infos dans la bdd bd_cen (#Foncier)
# RNNP
sql = '''SELECT DISTINCT ON (t1.par_id,t1.geom, t1.dcntpa,t1.typprop_id,t1.ccopre,t1.ccosec,t1.dnupla)
t1.par_id,t1.geom, t1.dcntpa,t1.typprop_id,t1.ccopre,
t1.ccosec,t1.dnupla::text, STRING_AGG(t6.gtoper::text,',') gtoper, STRING_AGG(t6.ccogrm::text,',') ccogrm,
STRING_AGG(CONCAT(t6.gtoper::text||COALESCE('_'||t6.ccogrm::text,'')),',') type_prop, STRING_AGG(t6.ddenom::text,',') ddenom
FROM {sch}."{t1}" t1
JOIN ({sch}."lots_38" t2
JOIN ({sch}."cadastre_38" t3
JOIN ({sch}."cptprop_38" t4
JOIN ({sch}."r_prop_cptprop_38" t5
JOIN {sch}."proprios_38" t6 USING (dnuper))
USING (dnupro))
USING (dnupro))
USING (lot_id))
USING (par_id)
WHERE par_id IN ('{par_ids}') GROUP BY 1,2,3,4,5,6'''.format(
sch='cadastre',
t1='parcelles_38',
par_ids="','".join(lst_par_id_rnnp))
parca1 = gpd.read_postgis(text(sql),con=con_fon)
parca1.to_file('/home/colas/Documents/tmp/FONCIER_FEDE/PLATIERE/parcelles_RNNP_tofede.gpkg', driver='GPKG')
# Récupération des infos dans la bdd bd_cen (#Foncier)
sql = '''SELECT DISTINCT ON (t1.par_id,t1.geom, t1.dcntpa,t1.typprop_id,t1.ccopre,t1.ccosec,t1.dnupla)
t1.par_id,t1.geom, t1.dcntpa,t1.typprop_id,t1.ccopre,
t1.ccosec,t1.dnupla::text, STRING_AGG(t6.gtoper::text,',') gtoper, STRING_AGG(t6.ccogrm::text,',') ccogrm,
STRING_AGG(CONCAT(t6.gtoper::text||COALESCE('_'||t6.ccogrm::text,'')),',') type_prop, STRING_AGG(t6.ddenom::text,',') ddenom
FROM {sch}."{t1}" t1
JOIN ({sch}."lots_38" t2
JOIN ({sch}."cadastre_38" t3
JOIN ({sch}."cptprop_38" t4
JOIN ({sch}."r_prop_cptprop_38" t5
JOIN {sch}."proprios_38" t6 USING (dnuper))
USING (dnupro))
USING (dnupro))
USING (lot_id))
USING (par_id)
WHERE par_id IN ('{par_ids}') GROUP BY 1,2,3,4,5,6'''.format(
sch='cadastre',
t1='parcelles_38',
par_ids="','".join(tofede.par_id))
parca2 = gpd.read_postgis(text(sql),con=con_fon)
cols_parca2 = parca2.columns
cols_join = [*cols_parcx[cols_parcx.isin(cols_parca2)]]
parcb = pd.concat([
parca1,
parca2[~parca2.par_id.isin(parcx.par_id)],
parcx[cols_join]
])
# n = 10
# while n!= 0:
# parcb.type_prop = parcb.type_prop.replace(',,',',',regex=True)
# n = n-1
parcb.loc[parcb.type_prop.str.contains('2_0'),['type_prop','gtoper','ccogrm']] = ['2_0_MIX','2','0']
parcb.loc[parcb.type_prop.isin(['2_7,2_3,','2_7,2_3,2_3,']),['type_prop','gtoper','ccogrm']] = ['2_0_MIX','2','0']
parcb.loc[parcb.type_prop.str.contains('2_9'),['type_prop','gtoper','ccogrm']] = ['2_9_PUB','2','9']
parcb.loc[parcb.type_prop.str.contains('2_8'),['type_prop','gtoper','ccogrm']] = ['2_8_PRI','2','8']
parcb.loc[parcb.type_prop.str.contains('2_7'),['type_prop','gtoper','ccogrm']] = ['2_7_PRI','2','7']
parcb.loc[parcb.type_prop.str.contains('2_6'),['type_prop','gtoper','ccogrm']] = ['2_6_PRI','2','6']
parcb.loc[parcb.type_prop.str.contains('2_5'),['type_prop','gtoper','ccogrm']] = ['2_5_PUB','2','5']
parcb.loc[parcb.type_prop.str.contains('2_4'),['type_prop','gtoper','ccogrm']] = ['2_4_COM','2','4']
parcb.loc[parcb.type_prop.str.contains('2_3'),['type_prop','gtoper','ccogrm']] = ['2_3_PUB','2','3']
parcb.loc[parcb.type_prop.str.contains('2_2'),['type_prop','gtoper','ccogrm']] = ['2_2_PUB','2','2']
parcb.loc[parcb.type_prop.str.contains('2_1'),['type_prop','gtoper','ccogrm']] = ['2_1_PUB','2','1']
parcb.loc[parcb.type_prop.str.contains('1,1'),['type_prop','gtoper','ccogrm']] = ['1_PRI','1',None]
parcb.loc[parcb.type_prop == '1',['type_prop','gtoper','ccogrm']] = ['1_PRI','1',None]
lstterm_cen38 = ["CONSERVATOIRE D'ESPACES NATURELS ISERE","AVENIR CONSERVATOIRE DES ESPACES NATURELS DE L'ISERE","CONSERVATOIRE D'ESPACES NATURELS ISERE AVENIR DIT CEN ISERE"]
lstterm_cenra = ["CONSERVATOIRE RHONE ALPES DES ESPACES NATURELS","CONSERVATOIRE RH ALP ESPACES NATURELS"]
parcb.loc[parcb.ddenom.isin([*lstterm_cen38,*lstterm_cenra]),['type_prop','gtoper','ccogrm']] = ['3_CEN','3',None]
# parcb.loc[parcb.type_prop.str.len()==3,'gtoper'] = parcb[parcb.type_prop.str.len()==3].type_prop.str[0]
# parcb.loc[parcb.type_prop.str.len()==3,'ccogrm'] = parcb[parcb.type_prop.str.len()==3].type_prop.str[-1]
parcb.loc[parcb.dnupla.str.len()==1,'dnupla'] = '000' + parcb.loc[parcb.dnupla.str.len()==1,'dnupla']
parcb.loc[parcb.dnupla.str.len()==2,'dnupla'] = '00' + parcb.loc[parcb.dnupla.str.len()==2,'dnupla']
parcb.loc[parcb.dnupla.str.len()==3,'dnupla'] = '0' + parcb.loc[parcb.dnupla.str.len()==3,'dnupla']
parcb['id_proprietaire'] = 0
parcb.loc[parcb.ddenom.str.contains('|'.join(lstterm_cenra),na=False),'id_proprietaire'] = '28'
# Récupération des infos dans la VM cadastre (#Foncier)
# sql = '''SELECT geo_parcelle, substring(geo_parcelle from 1 for 2)||substring(geo_parcelle from 4 for 12) par_id, geom
# FROM "38_202207".geo_parcelle'''
# 38517000AW0032 # par_id
# 380517000AW0032 # geo_parcelle
sql = '''SELECT
substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) par_id,
t1.annee, t1.update_dat, t1.geom
FROM "{sch}"."{t1}" t1
WHERE substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) IN ('{par_ids}')
'''.format(
sch='38_202207',
t1='v_geo_parcelle',
par_ids="','".join(tofede[~tofede.par_id.isin(parcx.par_id)].par_id))
parcc0 = gpd.read_postgis(text(sql),con=con_vm)
sql = '''SELECT
substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) par_id,
t1.annee, t1.update_dat, t1.geom
FROM "{sch}"."{t1}" t1
WHERE substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) IN ('{par_ids}')
'''.format(
sch='26_202007',
t1='v_geo_parcelle',
par_ids="','".join(tofede[~tofede.par_id.isin(parcx.par_id)].par_id))
parcc1 = gpd.read_postgis(text(sql),con=con_vm)
cols_parcc = parcc0.columns
cols_join = [*cols_parcx[cols_parcx.isin(cols_parcc)]]
parcc = pd.concat([parcc0,parcc1,parcx[cols_join]])
parcc['source_geom_parc_annee'] = parcc.annee
parcc['date_maj_parcelle'] = parcc.update_dat
l1 = dfgere.columns[dfgere.columns.isin(lst_cols_parc)]
l2 = tofede.columns[tofede.columns.isin(lst_cols_parc)]
l3 = parcb.columns[parcb.columns.isin(lst_cols_parc)]
l4 = parcc.columns[parcc.columns.isin(lst_cols_parc)]
p1 = dfgere[['par_id',*l1]].copy()
p2 = tofede[['par_id',*l2,'geom']].copy()
p3 = parcb[['par_id',*l3]].copy()
p4 = parcc[['par_id',*l4]].copy()
dfs = [p1,p2,p3,p4]
df_parcs = reduce(lambda left,right: pd.merge(left,right,on=['par_id'],
how='left'), dfs)
for par_id in df_parcs[df_parcs.par_id.duplicated(keep=False)].par_id.unique():
x = df_parcs[df_parcs.par_id==par_id].shape[0]
df_parcs.loc[df_parcs.par_id==par_id,'surf_parc_maitrise_m2'] = round(df_parcs.loc[df_parcs.par_id==par_id,'surf_parc_maitrise_m2']/x).astype(int)
df_parcs.date_maj_parcelle = df_parcs.date_maj_parcelle.astype(str)
gdf_tofede = df_parcs[['id_site_fcen_parc','num_parcelle','date_maj_parcelle','geom']].copy()
gdf_tofede = gdf_tofede.set_geometry('geom', crs=2154)
gdf_tofede.rename(columns={
'id_site_fcen_parc':'id_fcen',
'num_parcelle':'num_parc',
'date_maj_parcelle':'date_maj_p'
},inplace=True)
gdf_tofede.to_file('/home/colas/Documents/tmp/FONCIER_FEDE/SEND/Parcelles_CEN_38_%i.shp' % annee)
df_tofede = df_parcs[lst_cols_parc].copy()
df_tofede.loc[df_tofede.date_debut_ore.isna(),'date_debut_ore']
df_tofede.to_csv('/home/colas/Documents/tmp/FONCIER_FEDE/SEND/Parcelles_CEN_38_%i.csv' % annee)
# parcb.loc[(parcb.ddenom.str.contains("AVENIR CONSERVATOIRE DES ESPACES NATURELS DE L'ISERE",case=False))].ddenom.unique()
# parcb.loc[(parcb.ddenom.str.contains('CONSERVATOIRE|CEN',case=False))&(parcb.ddenom.str.contains('ISERE',case=False))].ddenom.unique()
# parcb.loc[parcb.gtoper.isin(['1,1','2,1','1,2','1,1,1']),'gtoper'] = '1'
# parcb.loc[parcb.gtoper.isin(['1,1,1,1','2,1,1,1,1,1,1','2,1,1,1,1,1,1,1,1,1,1,1,1,1,1','2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1']),'gtoper'] = '1'
# parcb.loc[parcb.gtoper.isin(['2,2']),'gtoper'] = '2'
# parcb.loc[parcb.gtoper=='1','ccogrm'] = None
parcb[parcb.par_id.duplicated(keep=False)]
dic_col = {
'code_dep' : 'insee_dep',
'section' : 'num_section',
'num_unique_v2' : 'num_parcelle', # ????
}
dic_parc = {
'id':'id_site_cen_parc',
}
lst_prop = ['Acquis par le CEN ou CREN','ACQUIS&CONVENTION', 'PARTIELLEMENT MAITRISE']
lst_conv = ["CONVENTION D'USAGE",'CONVENTION']
lst_gere = ['BAIL EMPHYTEOTIQUE', *lst_conv, *lst_prop]
sql = '''SELECT DISTINCT
t1.par_id,t1.geom, t1.dcntpa,t1.typprop_id,t1.ccopre,
t1.ccosec,t1.dnupla::text, t6.gtoper::text, t6.ccogrm::text
FROM {sch}."{t1}" t1
JOIN ({sch}."lots_38" t2
JOIN ({sch}."cadastre_38" t3
JOIN ({sch}."cptprop_38" t4
JOIN ({sch}."r_prop_cptprop_38" t5
JOIN {sch}."proprios_38" t6 USING (dnuper))
USING (dnupro))
USING (dnupro))
USING (lot_id))
USING (par_id)
WHERE ST_Within(t1.geom, 'SRID=2154;{geom}') '''.format(
sch='cadastre',
t1='parcelles_38',
geom=dffonc.unary_union)
sql = '''SELECT DISTINCT
t1.par_id,t1.geom, t1.dcntpa,t1.typprop_id,t1.ccopre,
t1.ccosec,t1.dnupla::text, t6.gtoper::text, t6.ccogrm::text
FROM {sch}."{t1}" t1
JOIN ({sch}."lots_38" t2
JOIN ({sch}."cadastre_38" t3
JOIN ({sch}."cptprop_38" t4
JOIN ({sch}."r_prop_cptprop_38" t5
JOIN {sch}."proprios_38" t6 USING (dnuper))
USING (dnupro))
USING (dnupro))
USING (lot_id))
USING (par_id)
WHERE ST_Intersects(t1.geom, 'SRID=2154;{geom}') '''.format(
sch='cadastre',
t1='parcelles_38',
geom=dffonc.unary_union)
sql = sql + " AND t1.par_id LIKE ANY (array['%s']);"% "','".join(dffonc[~dffonc.par_id.isna()].par_id)
parc = gpd.read_postgis(sql=text(sql),geom_col='geom',con=con_fon)
parc.loc[parc.dnupla.str.len()==1,'dnupla'] = '000' + parc.loc[parc.dnupla.str.len()==1,'dnupla']
parc.loc[parc.dnupla.str.len()==2,'dnupla'] = '00' + parc.loc[parc.dnupla.str.len()==2,'dnupla']
parc.loc[parc.dnupla.str.len()==3,'dnupla'] = '0' + parc.loc[parc.dnupla.str.len()==3,'dnupla']
# parc = gpd.read_file('/home/colas/Documents/tmp/FONCIER_FEDE/extract_parc_for_fede.gpkg')
# parc.rename_geometry('geom',inplace=True)
# parc.to_file('/home/colas/Documents/tmp/FONCIER_FEDE/extract_parc_for_fede.gpkg',driver='GPKG')
parc['semi_par_id'] = parc.par_id.str[:5] + parc.ccosec + parc.dnupla
######################
# HARMONISATION couche FONCIER
dffonc_b1 = dffonc_b5 = dffonc_b20 = dffonc_b50 = dffonc.copy()
dffonc_b1.geom = dffonc_b1.buffer(1)
dffonc_b20.geom = dffonc_b20.buffer(20)
dffonc_b50.geom = dffonc_b50.buffer(50)
dffonc_b5.geom = dffonc_b5.buffer(5)
tmp = gpd.sjoin(dffonc_b50[~dffonc_b50.numero.isna()],parc,op='contains',how='left')
tmp2 = tmp[tmp.numero == tmp.dnupla]
crsp = tmp2.id.tolist()
tmp[~tmp.id.isin(crsp)]
tmp[tmp.id.duplicated(keep=False)]
##############
##############
dfparc = parc[['par_id','gtoper','ccogrm','geom']]
dfgere = dffonc[dffonc.maitrise.isin(lst_gere)].copy() # parcelles gérée
dfgere['code_mfu1'] = None
dfgere['code_mfu2'] = None
dfgere.loc[dfgere.maitrise.isin(lst_prop),'code_mfu1'] = 'P'
dfgere.loc[dfgere.maitrise.isin(lst_conv),'code_mfu1'] = 'C'
dfgere.loc[dfgere.maitrise == 'BAIL EMPHYTEOTIQUE','code_mfu1'] = 'L'
dfgere.loc[dfgere.maitrise.isin(lst_prop),'code_mfu2'] = 'P1'
dfgere.loc[dfgere.maitrise.isin(lst_conv),'code_mfu2'] = 'C7'
dfgere.loc[dfgere.maitrise == 'BAIL EMPHYTEOTIQUE','code_mfu2'] = 'L1'
dfgere.loc[dfgere.maitrise == 'PARTIELLEMENT MAITRISE','bnd'] = 1
dfgere.loc[dfgere.maitrise != 'PARTIELLEMENT MAITRISE','bnd'] = 0
# Parcelles d'interventions
dftmp = dffonc[~dffonc.maitrise.isin(lst_gere)].copy() # couche temp des parcelles non-gérés
sittmp = dfsite[(dfsite.assist_cnv)&(dfsite.assist_col)] # sites en assistance
intersect = dftmp.intersects(sittmp.unary_union) # Intersection [couche temp] VS [sites assist]
dfintr = dftmp[intersect] # parcelles d'interventions
# Parcelles millitaire
dic = {
'ccosec':'num_section',
'par_id':'num_parcelle'
}
dfmili = dfsite[dfsite.type_site=='MILI'].copy()
sqlmil = "SELECT * FROM %s.%s WHERE ST_Contains('SRID=2154;%s',geom)" % ('cadastre', 'parcelles_38',dfmili.buffer(10).unary_union)
miliparc = gpd.read_postgis(sqlmil,con=con_fon)
miliparc['code_dep'] = miliparc.codcom.str[:2]
miliparc['code_com'] = miliparc.codcom.str[2:]
dfmili = miliparc[['par_id','geom','ccosec','code_dep','code_com']].copy()
dfmili.rename(columns=dic, inplace=True)
# All parcelles
df = pd.concat([dfgere,dfintr,dfmili], ignore_index=True)
keep_col = df.columns[df.columns.isin(lst_cols_parc)]
df = df[keep_col]
df.reset_index(drop=True, inplace=True)
sql = '''SELECT par_id,geom, dcbtpa,typprop_id FROM %s."%s" WHERE ST_Intersects(geom, 'SRID=2154;%s') ''' % ('cadastre', 'parcelles_38',df.representative_point().unary_union)
parc = gpd.read_postgis(sql=sql,geom_col='geom',con=con_fon)
sql = '''SELECT geo_parcelle,geo_section,idu,proprietaire,proprietaire_info,geom FROM %s."%s" WHERE ST_Intersects(geom, 'SRID=2154;%s') ''' % ('ref_cadastre', 'parcelle_info',df.representative_point().unary_union)
prop = gpd.read_postgis(sql=sql,geom_col='geom',con=con)
df['insee_com'] = df.code_dep + df.code_com
tmp = df[['id','geom']]
gpd.sjoin(df,parc[['par_id','geom']],how='left',op='intersects')
# df['num_parcelle'] =
df['surf_ore_m2'] = 0
df['date_debut_ore'] = None
df['date_fin_ore'] = None
lst_doc = []
df.loc[df.id.isin(lst_doc),'doc_foncier'] = 1
df.loc[~df.id.isin(lst_doc),'doc_foncier'] = 0
df.loc[df.doc_foncier == 1,'source_doc_foncier'] = 'CEN Isère'
df['pour_part'] = 0
df['domaine_public'] = 0
df['id_proprietaire'] = 0
df['parc_gestion_rnx'] = None # ?????????????????
df['source_geom_parc_nature'] = 2
df['source_geom_parc_annee'] = 2017
df['source_surf_parc'] = 1
# 3800010000A0001
# 38566AK00981000
# 596060000A0012
# 073490000C0103
###########
### Autre #
###########
df = pd.DataFrame()
df['id_cen'] = 14
df['id_site_cen'] = ''
0_FONCIER/TdB_FEDE/parcelles_TdB_fede2.py
+373
View File
@@ -0,0 +1,373 @@
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
from pycen import con_bdcen, con_cad
from os import getlogin
from os.path import join
from sqlalchemy.engine import URL
from sqlalchemy import create_engine
import geopandas as gpd
import pandas as pd
user = 'postgres'
pwd = 'postgres'
adr = '172.17.0.2'
base = 'postgres'
url = URL.create('postgresql+psycopg2',
username=user,
password=pwd,
host=adr,
database=base,
)
con_vm = create_engine(url)
annee = 2023
PATH = '/media/{login}/SRV/FICHIERS/OUTILS/BASES DE DONNEES/BILAN_FEDE_CEN/{an}/'
path_source = join(PATH,'Docs_SIG_joints{an}/').format(login=getlogin(),an=annee)
dico_data = 'Dico_DATA_sites_CEN_v%i_v1.xlsx' % annee
sheet_par = 'parcelles_cen_xx_%i' % annee
sheet_sit = 'sites_cen_xx_%i' % annee
# lv_cen = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='A:D',nrows=25)
# lv_typmaitr = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='F:K',nrows=23)
# lv_typmilie = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='M:N',nrows=16)
# lv_echelnum = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='P:Q',nrows=8)
# lv_typhab = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=13, usecols='P:Q',nrows=5)
# lv_sourcegeom = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='S:U',nrows=12)
# lv_lienres = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=20, usecols='M:N',nrows=4)
# lv_typprop = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=45, usecols='A:H',nrows=12)
# lv_codgeol = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=36, usecols='M:T',nrows=19)
lst_cols_parc = pd.read_excel(path_source+dico_data,sheet_name=sheet_par,header=0, usecols='F',nrows=26)\
['nom du champ'].tolist()
lst_prop = ['Acquis par le CEN ou CREN','ACQUIS&CONVENTION', 'PARTIELLEMENT MAITRISE','ACQUIS PAR LE CEN EN BND','ACQUIS PAR LE CEN ISERE']
lst_conv = ["CONVENTION D'USAGE",'CONVENTION']
lst_gere = ['BAIL EMPHYTEOTIQUE', *lst_conv, *lst_prop]
DICT_COLS = {
'id' : 'id_site_cen_parc',
'par_id' : 'num_parcelle',
'section': 'num_section',
'code_dep':'insee_dep',
'code_com':'insee_com',
}
def update_idproprietaire(df):
if 'id_proprietaire' in df.columns :
df['id_proprietaire'] = 0
df.loc[df.classif_prop.str.contains('CEN RA'),'id_proprietaire'] = 28
df.loc[df.classif_prop.str.contains('CEN ISERE'),'id_proprietaire'] = 14
df.id_proprietaire.fillna(0,inplace=True)
return df
def update_typeprop(df):
DICT = {
'Etat':'2_1_PUB',
'CEN RA - BND':'3_CEN',
'CEN RA':'3_CEN',
'CEN ISERE - BND':'3_CEN',
'Privé et entreprises':'1_PRI',
'Communes et comcom':'2_4_COM',
'Autres public et assimiliés (syndicats)':'2_0_MIX',
'Département':'2_3_PUB',
'Autres asso et fédérations':'2_6_PRI',
'FRUP':'3_CEN',
}
df['type_prop'] = df['classif_prop'].copy()
df['type_prop'].replace(DICT, inplace=True)
df.loc[
(df.classif_prop=='Privé et entreprises') &
(df.gtoper.str.contains('2')),
'type_prop'] = '2_0_MIX'
df.loc[
df.ccogrm == 9,
'type_prop'] = '2_9_PUB'
return df
def update_codemfu(df):
df.loc[df.maitrise.isin(lst_prop),'code_mfu1'] = 'P'
df.loc[df.maitrise.isin(lst_conv),'code_mfu1'] = 'C'
df.loc[df.maitrise == 'BAIL EMPHYTEOTIQUE','code_mfu1'] = 'L'
df.loc[df.maitrise.isin(lst_prop),'code_mfu2'] = 'P1'
df.loc[df.maitrise.isin(lst_conv),'code_mfu2'] = 'C7'
df.loc[df.maitrise == 'BAIL EMPHYTEOTIQUE','code_mfu2'] = 'L1'
df.loc[df.maitrise == 'ACCORD VERBAL','code_mfu1'] = 'C'
df.loc[df.maitrise == 'ACCORD VERBAL','code_mfu2'] = 'C17'
df.loc[df.maitrise == 'CONVENTION','code_mfu1'] = 'C'
df.loc[df.maitrise == 'CONVENTION','code_mfu2'] = 'C7'
df.loc[df.indivision=='USUFRUIT','code_mfu2'] = 'P2'
return df
def update_bnd(df):
df.loc[df.maitrise == 'PARTIELLEMENT MAITRISE','bnd'] = 1
df.loc[df.maitrise != 'PARTIELLEMENT MAITRISE','bnd'] = 0
df.loc[df.maitrise == 'ACQUIS&CONVENTION','bnd'] = 1
return df
def get_parc_milli():
sqlsit = '''
SELECT * FROM %s.%s
WHERE (date_fin is NULL OR date_fin >= '%i-01-01') AND type_site='MILI' ;'''%('sites','c_sites_zonages',annee)
df1 = gpd.read_postgis(sql=sqlsit,geom_col='geom',con=con_bdcen)
sql0 = '''SELECT
t1.geo_parcelle, substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) par_id,
t1.annee, t1.update_dat, t4.ccopre, t4.ccosec, t4.dcntpa, t1.geom, STRING_AGG(t3.gtoper::text,',') gtoper, STRING_AGG(t3.ccogrm::text,',') ccogrm,
STRING_AGG(CONCAT(t3.gtoper::text||COALESCE('_'||t3.ccogrm::text,'')),',') type_prop, STRING_AGG(t3.ddenom::text,',') ddenom
FROM "{sch}"."{t1}" t1
LEFT JOIN ("{sch}".parcelle_info t2
LEFT JOIN "{sch}".proprietaire t3 USING (comptecommunal))
USING (geo_parcelle)
LEFT JOIN "{sch}".parcelle t4 ON t1.geo_parcelle = t4.parcelle
'''.format(
sch='38_%i07'%(annee-1),
t1='v_geo_parcelle')
sql1 = '''
WHERE ST_INTERSECTS(t1.geom,'SRID={epsg};{poly}')
GROUP BY 1,2,3,4,5,6,7,8
'''.format(epsg=2154,poly=df1.unary_union.buffer(-1))
sql = sql0 + sql1
df2 = gpd.read_postgis(sql,con_cad)
df2['code_dep'] = df2.par_id.str[:2]
df2['code_com'] = df2.par_id.str[2:5]
df2['code_mfu1'] = 'C'
df2['code_mfu2'] = 'C12'
df2['section'] = df2.ccosec.copy()
df2['id_site_cen_parc'] = 0
df2['bnd'] = 0
df2['mesure_compens'] = 0
df2['pour_part'] = 0
return df2
def update_mc(df):
sqlsit = '''
SELECT * FROM %s.%s
WHERE (date_fin is NULL OR date_fin >= '%i-01-01') AND type_site='MC' ;'''%('sites','c_sites_zonages',annee)
df1 = gpd.read_postgis(sql=sqlsit,geom_col='geom',con=con_bdcen)
is_mc = df.intersects(df1.unary_union)
df['mesure_compens'] = 0
df.loc[
(is_mc & df.maitrise.isin(lst_conv)
)|(df.Parc_MCE=='OUI'),
'mesure_compens'
] = 1
return df
def update_pourpart(df):
df['pour_part'] = 0
df.loc[
df.maitrise=='PARTIELLEMENT MAITRISE',
'pour_part'
] = 1
return df
def is_rnx(df):
sqlsit = '''
SELECT * FROM %s.%s
WHERE (date_fin is NULL OR date_fin >= '%i-01-01') AND type_site='RNN' ;'''%('sites','c_sites_zonages',annee)
df1 = gpd.read_postgis(sql=sqlsit,geom_col='geom',con=con_bdcen)
isrnx = df.intersects(df1.unary_union.buffer(-1))
df['parc_gestion_rnx'] = 0
df.loc[isrnx,'parc_gestion_rnx'] = 1
return df
def update_datemajparcelle(df,cad):
tmp = df.merge(
parcx[['par_id','update_dat']],
how='left',
left_on='num_parcelle',
right_on='par_id')
tmp.rename(columns={'update_dat':'date_maj_parcelle'},inplace=True)
return tmp
def modif_FRUP2023(df):
df = df[
~df.commentair.str.contains(
# 'sign&2023|acq&2023',
r'sign.*2023|acq.*2023',
na=False,
case=False,
regex=True)]
is_frup = df.maitrise=='FRUP'
df.loc[is_frup,'classif_prop'] = 'CEN ISERE'
df.loc[is_frup,'maitrise'] = 'ACQUIS PAR LE CEN ISERE'
return df
def complete_tab(df):
df['id_site_cen_parc'] = df['id_site_cen_parc'].astype(int)
df['id_site_fcen_parc'] = None
df['bnd'] = df['bnd'].astype(int)
df.loc[df.type_prop.str.contains(','),'type_prop'] = '2_0'
df['surf_ore_m2'] = 0
df['date_debut_ore'] = None
df['date_fin_ore'] = None
df['doc_foncier'] = 1
df['source_doc_foncier'] = 0
df['source_geom_parc_nature'] = 2
df['source_geom_parc_annee'] = annee - 1
df['echelle_num_parc'] = 0
df['source_surf_parc'] = 1
df = update_datemajparcelle(df,parcx)
df['domaine_public'] = 0
return df
if __name__ == "__main__":
shplastsend = gpd.read_file(
join(PATH.format(login=getlogin(),an=annee-1),'ENVOIE','Parcelles_CEN_38_2022.shp')
)
lastsend = pd.read_csv(PATH.format(login=getlogin(),an=annee-1)+'ENVOIE/Parcelles_CEN_38_2022.csv')
sql = '''
SELECT * FROM foncier.c_anim_foncier_V2
WHERE maitrise not in ('Acquis par le CDI','Acquis par la commune ou Comcom','CONVENTION DE PASSAGE') AND maitrise is not NULL
; '''
dfparc = gpd.read_postgis(sql,con_bdcen)
if annee == 2023:
dfparc = modif_FRUP2023(dfparc)
dfparc = update_codemfu(dfparc)
dfparc = update_bnd(dfparc)
dfparc = update_pourpart(dfparc)
dfparc = update_mc(dfparc)
dfparc = is_rnx(dfparc)
dfparc = update_idproprietaire(dfparc)
dfparc = update_typeprop(dfparc)
dfparc['surf_parc_maitrise_m2'] = dfparc.area
# COLUMNS :
# Index(['par_id', 'geom', 'dcntpa', 'typprop_id', 'ccopre', 'section', 'dnupla',
# 'nom_com', 'numero', 'code_dep', 'code_com', 'gtoper', 'ccogrm',
# 'type_prop', 'nom_proprio', 'id', 'Parc_MCE', 'docs_off_num',
# 'Nbr_lots_BND', 'partiel_conv_ou_acq', 'code_mfu1', 'code_mfu2', 'bnd',
# 'mesure_compens', 'pour_part', 'par_id_v2', 'maitrise', 'date_der_c',
# 'indivision', 'contact', 'avis_propr', 'commentair', 'nom_proprio_old',
# 'site_id', 'type_zone', 'last_upd_parc', 'classif_prop', 'date_debut',
# 'date_fin'],
# dtype='object')
#
tmp = dfparc.rename(columns=DICT_COLS).columns.isin(lst_cols_parc)
lst_col = dfparc.rename(columns=DICT_COLS).columns[tmp]
parc = dfparc.rename(columns=DICT_COLS)[lst_col].copy()
shp_parc = dfparc[['par_id','geom']].rename(columns=DICT_COLS).copy()
sql0 = '''SELECT
t1.geo_parcelle, substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) par_id,
t1.annee, t1.update_dat, t4.ccopre, t4.ccosec, t4.dcntpa, t1.geom, STRING_AGG(t3.gtoper::text,',') gtoper, STRING_AGG(t3.ccogrm::text,',') ccogrm,
STRING_AGG(CONCAT(t3.gtoper::text||COALESCE('_'||t3.ccogrm::text,'')),',') type_prop, STRING_AGG(t3.ddenom::text,',') ddenom
FROM "{sch}"."{t1}" t1
LEFT JOIN ("{sch}".parcelle_info t2
LEFT JOIN "{sch}".proprietaire t3 USING (comptecommunal))
USING (geo_parcelle)
LEFT JOIN "{sch}".parcelle t4 ON t1.geo_parcelle = t4.parcelle
'''.format(
sch='38_202207',
t1='v_geo_parcelle')
sql1 = '''WHERE substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) IN ('%s')
GROUP BY 1,2,3,4,5,6,7,8'''%"','".join(dfparc.par_id.tolist())
sql = sql0 + sql1
parcx = gpd.read_postgis(sql,con=con_cad)
noparcx = dfparc[~dfparc.par_id.isin(parcx.par_id)]
newparc = parc[~parc.num_parcelle.isin(lastsend.num_parcelle)].copy()
shp_newparc = shp_parc[~shp_parc.num_parcelle.isin(lastsend.num_parcelle)].copy()
newparcx = parcx[parcx.par_id.isin(shp_newparc.num_parcelle)].copy()
shp_newparc = shp_newparc.merge(newparcx[['par_id','update_dat']],how='left',left_on='num_parcelle',right_on='par_id')\
.drop(columns=['par_id'])\
.rename(columns={
'num_parcelle':'num_parc',
'update_dat':'date_maj_p'
})\
.rename_geometry('geometry')
newparc = complete_tab(newparc)
parc = complete_tab(parc)
# mili = get_parc_milli()
PARC = pd.concat([lastsend,newparc])
PARC.id_site_cen_parc = PARC.id_site_cen_parc.astype(int)
# Mise au format des dates
date_cols = PARC.columns[PARC.columns.str.contains('date')]
for c in date_cols:
PARC[c] = pd.to_datetime(PARC[c],yearfirst=True).dt.strftime('%Y/%m/%d')
# site2023[c] = site2023[c].dt.strftime('%Y/%m/%d')
SHP_PARC = gpd.GeoDataFrame(
pd.concat([shplastsend,shp_newparc]),
geometry='geometry',
crs=2154
)
SHP_PARC['date_maj_p'] = pd.to_datetime(SHP_PARC['date_maj_p'],yearfirst=True).dt.strftime('%Y/%m/%d')
SHP_PARC.geometry = SHP_PARC.buffer(0)
PARC.to_csv(
join(
PATH.format(login=getlogin(),an=annee),
'Recueil_data',
'parcelles_cen_38_2023.csv'),
index=False)
SHP_PARC.to_file(
join(
PATH.format(login=getlogin(),an=annee),
'Recueil_data',
'parcelles_cen_38_2023.shp'),
index=False)
0_FONCIER/TdB_FEDE/site_TdB_2024_complete.py
+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
from pycen import con_fon
import pandas as pd
dico_data = '/media/cgeier/SRV/FICHIERS/OUTILS/BASES DE DONNEES/BILAN_FEDE_CEN/2024/TDB2024_enquete_SIG/Dico_DATA_sites_CEN_v2024.xlsx'
bilan_2023 = '/media/cgeier/SRV/FICHIERS/OUTILS/BASES DE DONNEES/BILAN_FEDE_CEN/2024/TDB2024_enquete_SIG/DATA N-1/Sites_CEN_38_2023.csv'
dic = pd.read_excel(dico_data,sheet_name='sites_cen_xx_2024',header=0, usecols='F',nrows=50)
dic_head_name = dic.columns[0]
bil2023 = pd.read_csv(bilan_2023,sep=',',header=0,encoding='utf-8')
# Mise en forme des dates
date_cols = bil2023.columns[bil2023.columns.str.contains('date')]
bil2023[date_cols] = bil2023[date_cols].apply(pd.to_datetime)
# Mise en forme de la colonne remq_sensibilite
vm_site = pd.read_sql_table('vm_sites_cen_2024_csv',con_fon,'_tdbfcen')
dic_missing = (dic.loc[~dic[dic_head_name].isin(vm_site.columns),dic_head_name]
.tolist())
(bil2023[['id_site_cen',*dic_missing]]
.merge(vm_site[['id_site_cen']],how='inner',on='id_site_cen')
.to_sql(
'_sites_cen_2023_csv_complement',
con_fon,
schema='_tdbfcen',
if_exists='replace',
index=False))
0_FONCIER/TdB_FEDE/sites_TdB_fede.py
+389
View File
@@ -0,0 +1,389 @@
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
from os import getlogin,path,chdir
import sys
import pandas as pd
import geopandas as gpd
from sqlalchemy.engine import URL
from sqlalchemy import create_engine
from pycen import con_bdcen as con, con_fon, wfs
# NOT WORK
# SCRIPT_DIR = path.dirname(path.abspath(__file__))
# sys.path.append(path.dirname(SCRIPT_DIR))
# from . import get_siteCen_fede
annee = 2023
term_parcelle_conv = ['CONVENTION',"CONVENTION D'USAGE",'ACCORD VERBAL']
path_agri = '/media/cgeier/SRV/FICHIERS/OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/TRAVAUX/Couches de référence/'
agri_partenaire_surf = gpd.read_file(path_agri+'AGRIS_partenaires_surface exploitées.shp')
agri_partenaire = gpd.read_file(path_agri+'AGRIS_partenaires.shp')
id_rnx = ['RNGL','RNIP']
MAJ_COLNAME = {
'doc_gestion_annee_ini':'doc_gestion_date_ini',
'doc_gestion_annne_maj':'doc_gestion_date_maj',
'source_geom_site_annee':'source_geom_site_date',
'0':'surf_carto_habitat_m2',
'F':'geol_site_inpn',
}
def drop_specialchar(obj):
return obj.replace({
r'[’]':"'",
r'[àáâãäå]':'a',
r'[èéêë]':'e',
r'[ìíîï]':'i',
r'[òóôõö]':'o',
r'[ùúûü]':'u',
# r'[ ]':"",
r'[–]':"-"
},regex=True)
def get_site():
sqlsit = '''SELECT * FROM %s.%s'''%('sites','c_sites_zonages')
return gpd.read_postgis(sql=sqlsit,geom_col='geom',con=con)
def get_siteCen_fede(an=None):
sqlsit = '''SELECT * FROM %s.%s'''%('sites','c_sites_zonages')
if an :
sqlsit += ''' WHERE date_fin is NULL OR date_fin >= '%i-01-01' '''%an
sqlsit += ' order by code_site'
dfsite = gpd.read_postgis(sql=sqlsit,geom_col='geom',con=con)
# Isolation de tous les sites CEN (gestion/assist/mili)
dup = dfsite.code_site.duplicated(keep=False)
typzon = dfsite.type_zonage.isin(['ZI','DO','DH'])
return dfsite[
~dup | (dup & typzon)
]
def get_parcelleCen_fede(an=None):
sqlsit = '''SELECT * FROM %s.%s'''%('foncier','c_anim_foncier_v2')
if an :
sqlsit += '''
WHERE maitrise IN (
'ACCORD VERBAL','ACQUI PAR LE CEN EN BND','ACQUI PAR LE CEN ISERE',
'ACQUIS PAR LE CEN ISERE','ACQUIS&CONVENTION','Acquis par le CEN ou CREN',
'BAIL EMPHYTEOTIQUE','CONVENTION','CONVENTION D''USAGE','FRUP')
AND (commentair not ilike '%%{an}%%' OR commentair IS NULL)'''.format(an=an)
return gpd.read_postgis(sql=sqlsit,geom_col='geom',con=con)
def update_date_site2022(df):
df['date_crea_site'].replace({
'01/012005': '01/01/2005',
'01/01/98': '01/01/1998',
'01/01/03': '01/01/2003',
'09.05.2001': '09/05/2001',
'03/1986': '01/03/1986',
'01/01/94': '01/01/1994',
}, inplace=True)
df['date_crea_site'] = pd.to_datetime(
df['date_crea_site'],
# format='%Y/%m/%d',
yearfirst=True
)
return df
def update_date(serie):
DICT = {
'024/04/2019':'24/04/2019'
}
return pd.to_datetime(
serie.replace(DICT),
# format='%Y/%m/%d',
yearfirst=True
)
if __name__ == "__main__":
path_source = '/media/{login}/SRV/FICHIERS/OUTILS/BASES DE DONNEES/BILAN_FEDE_CEN/{an}/'.format(login=getlogin(),an=annee)
path_an_precedent = '/media/{login}/SRV/FICHIERS/OUTILS/BASES DE DONNEES/BILAN_FEDE_CEN/{an}/ENVOIE/'.format(login=getlogin(),an=annee-1)
tdb_sites_csv = 'Sites_CEN_38_%i.csv'
receuil_path = 'Recueil_data/'
receuil_datasite = 'TBD - receuil data SITES.xlsx'
shp_cnpe = 'primtresitecnpe/perimetre_site_CNPE_2019.shp'
# path_source = '/media/{login}/SRV/FICHIERS/OUTILS/BASES DE DONNEES/BILAN_FEDE_CEN/{an}/Docs_SIG_joints{an}/'.format(login=getlogin(),an=annee)
dico_data = 'Docs_SIG_joints{an}/Dico_DATA_sites_CEN_v{an}_v1.xlsx'.format(an=annee)
lv_typmilie = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs',header=1, usecols='M:N',nrows=16)
lv_typmilie.columns = ['code_type_milieu','libelle_type_milieu']
dfsite = get_siteCen_fede(annee-1)\
.sort_values('code_site')\
.reset_index()
dfsite.loc[(dfsite.code_site=='N2IP') & (dfsite.type_zonage=='DO'),'code_site'] = 'N2IP_DO'
dfsite.loc[(dfsite.code_site=='N2IP') & (dfsite.type_zonage=='DH'),'code_site'] = 'N2IP_DH'
dfparc = get_parcelleCen_fede(annee)
sitefcen = dfsite[dfsite.intersects(dfparc.unary_union)]
parconv = dfparc.maitrise.isin(term_parcelle_conv)
site_rnx = dfsite[dfsite.code_site.isin(id_rnx)].copy()
n2000_zps = wfs.get_wfs(
url = "https://ws.carmencarto.fr/WMS/119/fxx_inpn?",
layer = "Zones_de_protection_speciale",
bbox = dfsite.unary_union )
# n2000_zps = wfs.get_wfs(url="https://ws.carmencarto.fr/WMS/119/fxx_inpn?",layer="Zones_de_protection_speciale")
n2000_zsc = wfs.get_wfs(
url="https://ws.carmencarto.fr/WMS/119/fxx_inpn?",
layer="Sites_d_importance_communautaire_JOUE__ZSC_SIC_",
bbox=dfsite.unary_union )
site2022 = pd.read_csv(path_an_precedent+tdb_sites_csv % (annee-1))\
.rename(columns=MAJ_COLNAME)\
.replace({'id_site_cen':{'COMB':'COCO'}})
site2022 = update_date_site2022(site2022)
site2023 = site2022[site2022.id_site_cen.isin(dfsite.code_site)]\
.reset_index(drop=True)
recueil_site = pd.read_excel(path_source+receuil_path+receuil_datasite,header=0,skiprows=[1])\
.replace({'Null':None})
recueil_site['doc_gestion_date_ini'] = update_date(recueil_site['doc_gestion_date_ini'])
recueil_site['doc_gestion_date_maj'] = update_date(recueil_site['doc_gestion_date_maj'])
recueil_site['doc_gestion_date_fin'] = update_date(recueil_site['doc_gestion_date_fin'])
test_add = ~recueil_site.id_site.isin(site2023.id_site_cen)
add_site = recueil_site[test_add].rename(columns={
'id_site':'id_site_cen'
})
site2023 = pd.concat([site2023,add_site])
# site_exploit_agri = dfsite.intersects(agri_partenaire.unary_union)
site_exploit_agri_surf = dfsite.intersects(agri_partenaire_surf.unary_union)
# Test d'identification des nouveaux sites
sitenew = ~dfsite.code_site.isin(site2022.id_site_cen)
if sitenew.any():
print('WARNING : de nouveaux sites sont apparus cette année :\n\t%s'%str(tuple(dfsite[sitenew].code_site)))
intersect_acqu = dfsite[sitenew].intersects(dfparc[~parconv].unary_union)
intersect_conv = dfsite[sitenew].intersects(dfparc[parconv].unary_union)
if intersect_acqu.any():
print(' Le(s) nouveau(x) site(s) intersect(ent) des parcelles acquises : \n\t%s'%str(tuple(dfsite[sitenew&intersect_acqu].code_site)))
else:
print(' Le(s) nouveau(x) site(s) ne contien(nen)t pas de parcelles acquises ...')
if intersect_conv.any():
print(' Le(s) nouveau(x) site(s) intersect(ent) des parcelles conventionnées : \n\t%s'%str(tuple(dfsite[sitenew&intersect_conv].code_site)))
else:
print(' Le(s) nouveau(x) site(s) ne contien(nen)t pas de parcelles conventionnées ...')
dfsite = dfsite[~sitenew].copy()\
.reset_index()
else :
print("Pas de nouveaux sites pour l'année %i."%annee)
site2023.sort_values('id_site_cen',inplace=True)
recueil_site.sort_values('id_site',inplace=True)
dfsite.sort_values('code_site',inplace=True)
# nom_site
site2023['nom_site'] = [
dfsite[dfsite.code_site==x].nom_site.values[0]
if x in [*dfsite.code_site] else None
for x in site2023.id_site_cen
]
site2023['nom_site'] = drop_specialchar(site2023['nom_site'].str.lower())
site2023['nom_site'] = site2023['nom_site'].str.upper()
# site_rnx_surface_m2
site2023.site_rnx_surface_m2 = dfsite\
.intersection(site_rnx.unary_union)\
.area.astype(int)
# ens (A FAIRE)
site2023['ens'] = site2023['ens'].fillna(1).astype(int)
# site_cdl
site2023.site_cdl = 0
# n2000_directive
site2023 = site2023.merge(dfsite[['code_site','geom']],how='left',left_on='id_site_cen',right_on='code_site')\
.drop(columns=['code_site'])
cnpe = gpd.read_file(path_source+receuil_path+shp_cnpe)
# site2023.loc[site2023.id_site_cen=='CNPE','geom'] = cnpe.iloc[0].geometry
site2023 = gpd.GeoDataFrame(site2023,geometry='geom',crs=2154)
site2023['is_zps'] = site2023.intersects(n2000_zps.unary_union)
site2023['is_zsc'] = site2023.intersects(n2000_zsc.unary_union)
site2023['n2000'] = site2023[['is_zps','is_zsc']].sum(axis=1)
site2023.loc[site2023.n2000==2,'n2000_directive'] = 'ZPS_ZSC'
site2023.loc[site2023.n2000==0,'n2000_directive'] = '0'
site2023.loc[site2023.is_zps & (site2023.n2000!=2),'n2000_directive'] = 'ZPS'
site2023.loc[site2023.is_zsc & (site2023.n2000!=2),'n2000_directive'] = 'ZSC'
site2023.drop(columns=['is_zps','is_zsc','n2000'],inplace=True)
# n2000_surface_m2
site2023['n2000_surface_m2'] = site2023\
.intersection(pd.concat([n2000_zps,n2000_zsc]).unary_union)\
.area.astype(int)
# terrain_militaire
site2023['terrain_militaire'] = (dfsite.type_site == 'MILI').astype(int)
# site_marin
site2023['site_marin'] = 0
# nb_contrat_agri
site2023['nb_contrat_agri'] = site2023['nb_contrat_agri'].fillna(0)
# nb_agri
site2023['nb_agri'] = site2023['nb_agri']\
.replace({'?':0})\
.fillna(0)\
.astype(int)
# surf_contra_m2
site2023['surf_contra_m2'] = site2023['surf_contra_m2'].fillna(0)
# code_milieu_princ
dict_milieu = dict(zip(lv_typmilie.libelle_type_milieu,lv_typmilie.code_type_milieu))
dfsite['milieux'].replace({'Gîtes à chiroptères et milieux souterrains':'Gîtes à chiroptères'}, inplace=True)
dfsite['code_milieu'] = dfsite.milieux.str.lower().replace(dict_milieu)
site2023['code_milieu_princ'] = [
dfsite[dfsite.code_site==x].code_milieu.values[0] if x in [*dfsite.code_site] else None for x in site2023.id_site_cen
]
# nature_site_inpn
site2023['nature_site_inpn'] = 'N'
# geol_site_inpn
site2023['geol_site_inpn'] = 'N'
# code_geol
site2023['code_geol'] = 21
# carto_habitats
site2023['carto_habitats'] = site2023['carto_habitats'].fillna(0)
# typo_carto_habitat
site2023['typo_carto_habitat'] = site2023['typo_carto_habitat'].fillna(0)
# surf_carto_habitat_m2
site2023['surf_carto_habitat_m2'] = site2023['surf_carto_habitat_m2'].fillna(0)
# date_crea_site
datemiss = site2023['date_crea_site'].isnull()
for s in site2023[datemiss].id_site_cen:
if s not in dfsite.code_site : pass
site2023.loc[site2023.id_site_cen==s,'date_crea_site'] = dfsite[dfsite.code_site==s].date_ajout
site2023['date_crea_site'] = pd.to_datetime(site2023['date_crea_site'],yearfirst=True, format='%Y/%m/%d')
# date_maj_site
site2023['date_maj_site'] = dfsite.date_maj.copy()
# nature_perimetre
site2023['nature_perimetre'] = site2023['nature_perimetre'].fillna(0)
# source_geom_site_nature
site2023['source_geom_site_nature'] = site2023['source_geom_site_nature'].fillna(0)
# source_geom_site_date
site2023['source_geom_site_date']
# echelle_num_site
site2023['echelle_num_site'] = site2023['echelle_num_site'].fillna(0)
# precision_num_site
site2023['precision_num_site'] = site2023['precision_num_site'].fillna('NE')
# gestionnaire_site
gestmiss = site2023['gestionnaire_site'].isna()
site2023.loc[gestmiss,'gestionnaire_site'] = 'CEN Isère'
# operateur
opmiss = site2023['operateur'].isna()
site2023.loc[opmiss,'operateur'] = 'FCEN'
# surf_libre_evolution_m2
site2023['surf_libre_evolution_m2'] = site2023['surf_libre_evolution_m2'].fillna(0)
# doc_gestion_presence
site2023['doc_gestion_presence'] = site2023['doc_gestion_presence'].fillna(0)
# doc_gestion_nom
site2023['doc_gestion_nom']
# doc_gestion_evaluation
site2023['doc_gestion_evaluation']
# doc_gestion_date_ini
# site2023['doc_gestion_date_ini'] = pd.to_datetime(site2023['doc_gestion_date_ini'],yearfirst=True)
site2023['doc_gestion_date_ini'] = update_date(site2023['doc_gestion_date_ini'])
# doc_gestion_date_maj
# site2023['doc_gestion_date_maj'] = pd.to_datetime(site2023['doc_gestion_date_maj'],yearfirst=True)
site2023['doc_gestion_date_maj'] = update_date(site2023['doc_gestion_date_maj'])
# doc_gestion_date_fin
# site2023['doc_gestion_date_fin'] = pd.to_datetime(site2023['doc_gestion_date_fin'],yearfirst=True)
site2023['doc_gestion_date_fin'] = update_date(site2023['doc_gestion_date_fin'])
# surf_doc_gestion_m2
site2023['surf_doc_gestion_m2'] = site2023['surf_doc_gestion_m2'].fillna(0)
# url_fiche_inpn
site2023['url_fiche_inpn']
# url_fiche_cen
site2023['url_fiche_cen']
# doc_justif_admin
site2023['doc_justif_admin']
# ouverture_public
site2023['ouverture_public']
# description_site
site2023['description_site']
# url_site_photo
site2023['url_site_photo']
# sensibilite
site2023['sensibilite']
# remq_sensibilite
site2023['remq_sensibilite']
site2023.sort_values('id_site_cen',inplace=True)
recueil_site.sort_values('id_site',inplace=True)
for c in recueil_site.columns:
if c not in site2023.columns: pass
notna = recueil_site[c].notna()
lstsite = recueil_site[notna].id_site.tolist()
s_2023 = site2023.id_site_cen.isin(lstsite)
r_2023 = recueil_site.id_site.isin(lstsite)
site2023.loc[s_2023,c] = recueil_site[r_2023][c]
# test_add = ~recueil_site.id_site.isin(site2023.id_site_cen)
# add_site = recueil_site[test_add].rename(columns={
# 'id_site':'id_site_cen'
# })
# SITE2023 = pd.concat([site2023,recueil_site[test_add]])
for c in site2023.columns:
if site2023[c].dtype==float:
try:
site2023[c] = site2023[c].astype(int)
except:
pass
# Mise au format des dates
date_cols = site2023.columns[site2023.columns.str.contains('date')]
for c in date_cols:
site2023[c] = pd.to_datetime(site2023[c],yearfirst=True).dt.strftime('%Y/%m/%d')
# site2023[c] = site2023[c].dt.strftime('%Y/%m/%d')
shp2023 = site2023[['id_site_cen','id_site_fcen','date_crea_site','date_maj_site','geom']].copy()
shp2023['date_maj_s'] = [
shp2023.date_maj_site[i] if pd.notna(shp2023.date_maj_site[i]) else shp2023.date_crea_site[i] for i in shp2023.index
]
shp2023.drop(columns=['date_crea_site','date_maj_site'],inplace=True)
shp2023.rename(columns={'id_site_fcen':'id_fcen'},inplace=True)
# Ecriture des fichiers finaux
site2023.drop(columns=['geom','id_site','gestionnaire']).to_csv(path_source+receuil_path+tdb_sites_csv%annee,index=False)
shp2023.to_file(
(path_source+receuil_path+tdb_sites_csv[:-3]+'shp')%annee,
index=False)
# from pycen import update_to_sql,
# site = get_site()
# darp = site.code_site.isin(['DARN','PEYR'])
# zi = site.type_zonage=='ZI'
# zo = site.type_zonage=='ZO'
# DARP = gpd.GeoDataFrame({
# 'code_site':['DARP','DARP'],
# 'type_zonage': ['ZI','ZO'],
# },
# geometry=[
# site[darp&zi].unary_union,site[darp&zo].unary_union
# ],crs=2154)\
# .rename_geometry('geom')
# DARP['surface_ha'] = round(DARP.area / 10000,2)
# update_to_sql(cnpe,con,'c_sites_zonages','sites','code_site')
0_FONCIER/add_c_anim_foncier_v2.py
+91
View File
@@ -0,0 +1,91 @@
from pycen import con_bdcen,con_fon,update_to_sql
import geopandas as gpd
from datetime import datetime as dt
# GET parcelles gérées à jour
sql = "SELECT * FROM sites.c_sites_zonages WHERE code_site = 'LEZE'"
sql = "SELECT * FROM foncier.c_anim_foncier_v2"
parc_site = gpd.read_postgis(sql,con_bdcen)
parc_site[(~parc_site.classif_prop.str.contains('cen',case=False))&(parc_site.maitrise.str.contains('acqui',case=False))].to_csv('/media/cgeier/SRV/FICHIERS/TRANSFERTS-EQUIPE/AB/verif_anim_foncier_v2.csv',index=False)
# parc_site.loc[parc_site.maitrise=='ACQUI PAR LE CEN ISERE','maitrise'] = 'ACQUIS PAR LE CEN ISERE'
# update_to_sql(
# parc_site[['par_id','maitrise']],con_bdcen,'c_anim_foncier_v2','foncier','par_id'
# )
# GET sites
sql = "SELECT * FROM sites.sites"
site_fon = gpd.read_postgis(sql,con_fon)
site_zi = site_fon[site_fon.site_id.str.endswith('ZI')]
site_zo = site_fon[site_fon.site_id.str.endswith('ZO')]
# GET cadastre from bd_cen
sql = '''
WITH prop AS (
SELECT
dnuper,gtoper,ccogrm,ddenom
FROM {sch}."proprios"
ORDER BY 2,3,4
)
SELECT --DISTINCT ON (t1.par_id,t1.geom, t1.dcntpa,t1.typprop_id,t1.ccopre,t1.ccosec,t1.dnupla)
t1.par_id,
t1.geom,
t1.dcntpa,
t1.typprop_id,
t1.ccopre,
t1.ccosec "section",
t1.dnupla::text,
c.nom nom_com,
substring(t1.par_id from 12 for 4) "numero",
substring(t1.codcom from 1 for 2) code_dep,
substring(t1.codcom from 3 for 3) code_com,
STRING_AGG(distinct t6.gtoper::text,',') gtoper,
STRING_AGG(distinct t6.ccogrm::text,',') ccogrm,
STRING_AGG(distinct CONCAT(TRIM(t6.gtoper::text)||COALESCE('_'||TRIM(t6.ccogrm::text),'')),',') type_prop,
--t6.dnuper,
STRING_AGG(distinct t6.ddenom::text,', ') nom_proprio
FROM {sch}."{t1}" t1
JOIN ({sch}."lots" t2
JOIN ({sch}."cadastre" t3
JOIN ({sch}."cptprop" t4
JOIN ({sch}."r_prop_cptprop" t5
JOIN prop t6 USING (dnuper))
USING (dnupro))
USING (dnupro))
USING (lot_id))
USING (par_id)
JOIN administratif.communes c on t1.codcom = c.code_insee '''.format(
sch='cadastre',
t1='parcelles'
)
sql_grp = '''GROUP BY 1,2,3,4,5,6,8'''
# Parcelles gestion
csv_parc = gpd.pd.read_csv('/home/colas/Documents/9_PROJETS/0_FONCIER/CEN38/LEZE_parcelles_a_rajouter_CSV.csv')
csv_parc['par_id'] = csv_parc.geo_parcel.str[:2]+csv_parc.geo_parcel.str[3:]
lst_parc = csv_parc.par_id.tolist()
sql_whr = "WHERE t1.par_id IN ('{par_ids}') ".format(par_ids="','".join(lst_parc))
sql_gst = sql + sql_whr + sql_grp
parc = gpd.read_postgis(sql_gst,con_fon)
parc['last_upd_parc'] = dt.now().date().isoformat()
parc.loc[parc.type_prop.str.startswith(','),'type_prop'] = parc[parc.type_prop.str.startswith(',')].type_prop.str[1:]
parc['classif_prop'] = 'Privé et entreprises'
parc.loc[parc.type_prop=='1','classif_prop'] = 'Privé et entreprises'
parc2 = parc.sjoin(site_fon[['site_id','geom']],how='left')\
.replace({'site_id':{'_ZI|_ZO':''}},regex=True)\
.drop(columns=['index_right'])\
.drop_duplicates()\
.reset_index(drop=True)
# Envoie bd-cen-38.foncier.c_anim_foncier_v2
parc2.to_postgis(
name = 'c_anim_foncier_v2',
con = con_bdcen,
schema = 'foncier',
if_exists = 'append',
index=False
)
0_FONCIER/config_insert_cadastre.py
+118
View File
@@ -0,0 +1,118 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Nom : : config_insert_table.py
#Description : Insertion des données cadastrales à la base <foncier> après de sa création.
#Copyright : 2021, CEN38
#Auteur : Colas Geier
#Version : 1.0
DICT_TAB = [{
'table_in' : 'proprietaire', # Table source qui provient de la sortie du plugin cadastre de qgis
'index_tab': 'proprietaire', # Pkey de la table source
'columns_in': ['ccodep', 'dnuper', 'ccoqua', 'ddenom', 'jdatnss', 'dldnss', 'dsglpm', 'dlign3', 'dlign4', 'dlign5', 'dlign6', 'dnatpr', 'gtoper', 'ccogrm',
'ccocom', 'dnupro', 'dnomlp', 'dprnlp', 'epxnee', 'dnomcp', 'dprncp', 'ccodro', 'ccodem'],
'table_out': [{
'name': 'cptprop{}'.format(dpt_nom_tab),
'geom': None,
'drop_escape': False, # Supprime les champs vides à l'intérieure des chaines de carractères
'columns_in': ['ccodep', 'ccocom', 'dnupro'], # Liste des columns à récupérer en entrée.
'columns_add': {'dnupro': ['ccodep', 'ccocom', 'dnupro']}, # Définition des champs composés devant être ajoutés
'unique': {'cols': ['dnupro'], 'keep': 'first'}, # Champs devant être uniques à l'intérieur de la table en sortie
'dict': None, # Dictionnaire pour renommer les champs {'ancien_nom1': 'nouveau_nom1', 'ancien_nom2': 'nouveau_nom2', ...}
'join': False
},{
'name': 'proprios{}'.format(dpt_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in': ['ccodep', 'dnuper', 'ccoqua', 'ddenom', 'jdatnss', 'dldnss', 'dsglpm', 'dlign3', 'dlign4', 'dlign5', 'dlign6', 'dnatpr', 'gtoper', 'ccogrm'],
'columns_add': {'dnuper': ['ccodep', 'dnuper']},
'unique': {'cols': ['dnuper'], 'keep': 'first'},
'dict': None,
'join': False
},{
'name': 'r_prop_cptprop{}'.format(dpt_nom_tab),
'geom': None,
'drop_escape': False,
'columns_in': ['ccodep', 'dnuper', 'ccocom', 'dnupro', 'dnomlp', 'dprnlp', 'epxnee', 'dnomcp', 'dprncp', 'ccodro', 'ccodem'],
'columns_add': {
'dnuper': ['ccodep', 'dnuper'],
'dnupro': ['ccodep', 'ccocom', 'dnupro']},
'unique': {'cols': ['dnupro', 'dnuper'], 'keep': 'first'},
'dict': None,
'join': False
},]
},{
'table_in' : 'parcelle',
'index_tab': 'parcelle',
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'ccovoi', 'dparpi', 'dcntpa', 'ccocomm', 'ccoprem', 'ccosecm', 'dnuplam', 'dvoilib', 'type_filiation'],
'table_out': [{
'name': 'vl{}'.format(dpt_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccovoi', 'dvoilib'],
'columns_add': {
'vl_id': ['ccodep', 'ccocom', 'ccovoi'],
'geom': None},
'unique': {'cols': ['vl_id'], 'keep': 'first'},
'dict': None,
'join': False
},{
'name': 'parcelles{}'.format(dpt_nom_tab),
'geom': {
'table_geom_in': 'geo_parcelle',
'index_geom': 'geo_parcelle'
},
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'ccovoi', 'dparpi', 'dcntpa', 'ccocomm', 'ccoprem', 'ccosecm', 'dnuplam', 'type_filiation'],
'columns_add': {
'par_id': ['ccodep', 'ccocom', 'ccopre','ccosec', 'dnupla'],
'codcom': ['ccodep', 'ccocom'],
'vl_id': ['ccodep', 'ccocom', 'ccovoi'],
'typprop_id': None },
'unique': False,
'dict': {'type_filiation': 'type'},
'join': False
},{
'name': 'lots{}'.format(dpt_nom_tab), # !!!!!! Ne trouve pas de parcelles sans lots (ex: 38357000AE0526)
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dcntpa'],
'columns_add': {
'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla'],
'par_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla'],
'dnulot': None, },
'unique': False,
'dict': {'dcntpa': 'dcntlo'},
'join': [{'bdd': 'out', 'table': 'parcelles{}'.format(dpt_nom_tab), 'on': ['par_id'], 'type': 'distinct'}]
},]
},{
'table_in' : 'lots',
'index_tab': 'lots',
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnupdl', 'dcntlo'],
'table_out': [{
'name': 'lots{}'.format(dpt_nom_tab), # !!!!!! parcelles avec lots: existe par_id NOT IN parcelles_73
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnupdl', 'dcntlo'],
'columns_add': {
'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'],
'par_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla'],},
'unique': {'cols': ['lot_id'], 'keep': 'first'},
'dict': None,
'join': False
},{
'name': 'lots_natcult{}'.format(dpt_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'],
'columns_add': {
'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'],},
'unique': {'cols': ['lot_id'], 'keep': 'first'},
'dict': None,
'join': [{ # ERROR ! 2 dclssf pour 1 lot_id
'bdd': 'in', 'table': 'suf', 'on': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'], 'type': 'merge',
'select_cols' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot','dsgrpf','cnatsp','dclssf','ccosub','dcntsf'],
}]
},]
}]
0_FONCIER/create_c_anim_foncier_v2.py
+187
View File
@@ -0,0 +1,187 @@
from pycen import con_bdcen,con_fon
import geopandas as gpd
from datetime import datetime as dt
# GET parcelles gérées à jour
sql = "SELECT * FROM parcelles_foncier_tofede"
parc_site = gpd.read_postgis(sql,con_bdcen)
parc_smrg = parc_site[[
'par_id','id','Parc_MCE','docs_off_num',
'Nbr_lots_BND','partiel_conv_ou_acq',
'code_mfu1','code_mfu2','bnd','mesure_compens',
'pour_part'
]]
# GET couche animation fonciere
sql = """SELECT
c.*,
substring(c.num_unique_v2 from 1 for 5) || substring(c.num_unique_v2 from 13 for 3) || substring(c.num_unique_v2 from 6 for 6) par_id_v2
FROM foncier.c_animation_fonciere_sites c"""
anim_fon = gpd.read_postgis(sql,con_bdcen)
anim_mrg = anim_fon[['par_id_v2',
'id','maitrise','date_der_c','indivision',
'contact','avis_propr','commentair','nom_propri'
]].rename(
columns={
'nom_propri':'nom_proprio_old'
}
)
# GET sites
sql = "SELECT * FROM sites.sites"
site_fon = gpd.read_postgis(sql,con_fon)
site_zi = site_fon[site_fon.site_id.str.endswith('ZI')]
site_zo = site_fon[site_fon.site_id.str.endswith('ZO')]
lst_pmili = parc_site[parc_site.id.isna()].par_id.tolist()
# 3880,3884,3885,3883,3886,3859,3860,3856,3858,3854
# par_id / lot_id
# '38222000ZB0013','38222000ZB0015','38222000ZB0016','38222000ZB0017','38222000ZB0023','38222000ZB0041','38222000ZB0042','38222000ZB0043','38222000ZB0058','38222000ZB0059'
# vl_id
# '3822200101','3822200126','3822200126','3822200126','3822200108','3822200106','3822200106','3822200106','3822200089','3822200089'
# GET cadastre from bd_cen
sql = '''
WITH prop AS (
SELECT
dnuper,gtoper,ccogrm,ddenom
FROM {sch}."proprios"
ORDER BY 2,3,4
)
SELECT --DISTINCT ON (t1.par_id,t1.geom, t1.dcntpa,t1.typprop_id,t1.ccopre,t1.ccosec,t1.dnupla)
t1.par_id,
t1.geom,
t1.dcntpa,
t1.typprop_id,
t1.ccopre,
t1.ccosec "section",
t1.dnupla::text,
c.nom nom_com,
substring(t1.par_id from 12 for 4) "numero",
substring(t1.codcom from 1 for 2) code_dep,
substring(t1.codcom from 3 for 3) code_com,
STRING_AGG(distinct t6.gtoper::text,',') gtoper,
STRING_AGG(distinct t6.ccogrm::text,',') ccogrm,
STRING_AGG(distinct CONCAT(TRIM(t6.gtoper::text)||COALESCE('_'||TRIM(t6.ccogrm::text),'')),',') type_prop,
--t6.dnuper,
STRING_AGG(distinct t6.ddenom::text,', ') nom_proprio
FROM {sch}."{t1}" t1
JOIN ({sch}."lots" t2
JOIN ({sch}."cadastre" t3
JOIN ({sch}."cptprop" t4
JOIN ({sch}."r_prop_cptprop" t5
JOIN prop t6 USING (dnuper))
USING (dnupro))
USING (dnupro))
USING (lot_id))
USING (par_id)
JOIN administratif.communes c on t1.codcom = c.code_insee '''.format(
sch='cadastre',
t1='parcelles'
)
sql_grp = '''GROUP BY 1,2,3,4,5,6,8'''
# Parcelles gestion
lst_parc = parc_site.par_id.tolist()
sql_whr = "WHERE t1.par_id IN ('{par_ids}') ".format(par_ids="','".join(lst_parc))
sql_gst = sql + sql_whr + sql_grp
parc_gst = gpd.read_postgis(sql_gst,con_fon)
parc_gst_tmp = parc_gst.merge(parc_smrg,on='par_id')
parc_gst_2 = parc_gst_tmp.merge(anim_mrg,on='id')
# parcelles sites, NON PRESENTES dans parcelles gérées
geo = site_fon.unary_union
sql_whr = """
WHERE ST_Intersects (t1.geom, 'SRID={epsg};{poly}')
AND t1.par_id NOT IN ('{par_ids}')""".format(
epsg = 2154,
poly = geo,
par_ids="','".join(lst_parc)
)
sql_veil = sql + sql_whr + sql_grp
parc_veil = gpd.read_postgis(sql_veil,con_fon)
parc_veil2 = parc_veil.merge(anim_mrg,left_on='par_id',right_on='par_id_v2',how='left')
# Parcelles Hors site
lst_id = [
*parc_veil2.id.dropna().astype(int).tolist(),
*parc_gst_2.id.dropna().astype(int).tolist()
]
lst_parid_HS = anim_fon[~anim_fon.id.isin(lst_id)].par_id_v2.dropna().tolist()
sql_whr = "WHERE t1.par_id IN ('{par_ids}') ".format(par_ids="','".join(lst_parid_HS))
sql_hs = sql + sql_whr + sql_grp
parc_hs = gpd.read_postgis(sql_hs,con_fon)
# parc_hs2 = parc_hs.merge(anim_mrg,left_on='par_id',right_on='par_id_v2')
parc_hs2 = parc_hs[parc_hs.is_valid].merge(anim_mrg,left_on='par_id',right_on='par_id_v2')
INVALID_parc_hs = parc_hs[~parc_hs.is_valid].merge(anim_mrg,left_on='par_id',right_on='par_id_v2')
# parc_gst_2[parc_gst_2.type_prop.str.startswith(',')].iloc[:,:15]
# parc_veil2[parc_veil2.type_prop.str.startswith(',')].iloc[:,:15]
# parc_hs2[parc_hs2.type_prop.str.startswith(',')]
# Merge parcelles
parc = gpd.pd.concat([parc_gst_2,parc_veil2,parc_hs2])\
.reset_index(drop=True)
parc.loc[parc.type_prop.str.startswith(','),'type_prop'] = parc[parc.type_prop.str.startswith(',')].type_prop.str[1:]
# Jointure id_site
parc2 = parc.sjoin(site_fon[['site_id','geom']],how='left')\
.replace({'site_id':{'_ZI|_ZO':''}},regex=True)\
.drop(columns=['index_right'])\
.drop_duplicates()\
.reset_index(drop=True)
# Drop manuel des doublons
parc_CRAS = ['381370000B0249','381370000B0241','381370000B0248','381370000B0250','381370000B0582']
parc_MONA = ['383100000A0693']
id_drop1 = parc2[parc2.par_id.isin(parc_CRAS) & (parc2.site_id != 'CRAS')].index
id_drop2 = parc2[parc2.par_id.isin(parc_MONA) & (parc2.site_id != 'MONA')].index
lst_id_drop = [*id_drop1,*id_drop2]
parc2.drop(lst_id_drop,inplace=True)
parc2.loc[parc2.par_id == '381370000B0566','site_id'] = 'CRAS'
# Jointure type_zone
parc2['zi'] = parc2.intersects(site_zi.unary_union)
parc2['zo'] = parc2.intersects(site_zo.unary_union)
parc2.loc[parc2.zi,'type_zone'] = 'ZI'
parc2.loc[parc2.zo & (parc2.type_zone != 'ZI'),'type_zone'] = 'ZO'
parc2.drop(columns = ['zi','zo'],inplace=True)
# PARC = gpd.pd.concat([parc2,parc_hs2])
parc2['last_upd_parc'] = dt.now().date().isoformat()
# Envoie bd-cen-38.foncier.c_anim_foncier_v2
parc2.to_postgis(
name = 'c_anim_foncier_v2',
con = con_bdcen,
schema = 'foncier',
if_exists = 'replace',
index=False
)
INVALID_parc_hs.drop(columns='geom').to_sql(
name = 'c_anim_foncier_v2',
con = con_bdcen,
schema = 'foncier',
if_exists = 'append',
index=False
)
sql = """
ALTER TABLE foncier.c_anim_foncier_v2 ADD PRIMARY KEY (par_id);
GRANT ALL ON TABLE foncier.c_anim_foncier_v2 TO abavarot;
GRANT ALL ON TABLE foncier.c_anim_foncier_v2 TO grp_admin;
GRANT ALL ON TABLE foncier.c_anim_foncier_v2 TO cen_admin;
GRANT SELECT ON TABLE foncier.c_anim_foncier_v2 TO grp_consult;
"""
with con_bdcen.begin() as cnx:
cnx.execute(sql)
0_FONCIER/create_foreign_zh.py
+126
View File
@@ -0,0 +1,126 @@
from pycen import con_fon
foreign_server = """
CREATE SERVER fdw_azalee
FOREIGN DATA WRAPPER postgres_fdw
OPTIONS (dbname 'azalee', host '91.134.194.221', port '5432')
"""
user_mapping = """
CREATE USER MAPPING
FOR cen_admin
SERVER fdw_azalee
OPTIONS (user 'cen_admin', password '#CEN38@venir')
"""
foreign_table = """
-- DROP FOREIGN TABLE inventaires.inventaire_zh;
CREATE FOREIGN TABLE inventaires.inventaire_zh (
site_code varchar(10) NULL,
nom varchar NULL,
auteur_site varchar NULL,
auteur_geom varchar NULL,
auteur_last_maj varchar NULL,
date_site date NULL,
date_geom date NULL,
date_last_maj date NULL,
type_milieu varchar NULL,
type_site varchar NULL,
typo_sdage varchar NULL,
rmq_site text NULL,
rmq_fct_majeur text NULL,
rmq_interet_patri text NULL,
rmq_bilan_menace text NULL,
rmq_orient_act text NULL,
rmq_usage_process text NULL,
code_cb varchar NULL,
lib_cb text NULL,
activ_hum varchar NULL,
impact varchar NULL,
"position" varchar NULL,
rmq_activ_hum text NULL,
connexion varchar NULL,
subm_orig varchar NULL,
subm_freq varchar NULL,
subm_etend varchar NULL,
fct_bio varchar NULL,
fct_hydro varchar NULL,
int_patri varchar NULL,
"val_socioEco" varchar NULL,
crit_delim varchar NULL,
crit_def_esp varchar NULL,
entree_eau_reg varchar NULL,
entree_eau_perm varchar NULL,
entree_eau_topo varchar NULL,
sortie_eau_reg varchar NULL,
sortie_eau_perm varchar NULL,
sortie_eau_topo varchar NULL,
geom public.geometry(geometry, 2154) NULL
)
SERVER fdw_azalee
OPTIONS (schema_name 'zones_humides', table_name 'v_zoneshumides');
-- Permissions
ALTER TABLE inventaires.inventaire_zh OWNER TO cen_admin;
GRANT ALL ON TABLE inventaires.inventaire_zh TO cen_admin;
"""
view_v_zoneshumides = """
-- inventaires.v_zoneshumides source
CREATE OR REPLACE VIEW inventaires.v_zoneshumides
AS SELECT inventaire_zh.site_code,
inventaire_zh.nom,
inventaire_zh.auteur_site,
inventaire_zh.auteur_geom,
inventaire_zh.auteur_last_maj,
inventaire_zh.date_site,
inventaire_zh.date_geom,
inventaire_zh.date_last_maj,
inventaire_zh.type_milieu,
inventaire_zh.type_site,
inventaire_zh.typo_sdage,
inventaire_zh.rmq_site,
inventaire_zh.rmq_fct_majeur,
inventaire_zh.rmq_interet_patri,
inventaire_zh.rmq_bilan_menace,
inventaire_zh.rmq_orient_act,
inventaire_zh.rmq_usage_process,
inventaire_zh.code_cb,
inventaire_zh.lib_cb,
inventaire_zh.activ_hum,
inventaire_zh.impact,
inventaire_zh."position",
inventaire_zh.rmq_activ_hum,
CASE
WHEN inventaire_zh.rmq_activ_hum ~~ '%remblai%'::text THEN 't'::text
ELSE 'f'::text
END AS remblais,
inventaire_zh.connexion,
inventaire_zh.subm_orig,
inventaire_zh.subm_freq,
inventaire_zh.subm_etend,
inventaire_zh.fct_bio,
inventaire_zh.fct_hydro,
inventaire_zh.int_patri,
inventaire_zh."val_socioEco",
inventaire_zh.crit_delim,
inventaire_zh.crit_def_esp,
inventaire_zh.entree_eau_reg,
inventaire_zh.entree_eau_perm,
inventaire_zh.entree_eau_topo,
inventaire_zh.sortie_eau_reg,
inventaire_zh.sortie_eau_perm,
inventaire_zh.sortie_eau_topo,
inventaire_zh.geom
FROM inventaires.inventaire_zh;
-- Permissions
ALTER TABLE inventaires.v_zoneshumides OWNER TO cen_admin;
GRANT ALL ON TABLE inventaires.v_zoneshumides TO cen_admin;
GRANT SELECT ON TABLE inventaires.v_zoneshumides TO grp_sig;
GRANT SELECT ON TABLE inventaires.v_zoneshumides TO cen_user;
"""
with con_fon.begin() as cnx:
cnx.execute(foreign_server)
cnx.execute(user_mapping)
cnx.execute(foreign_table)
cnx.execute(view_v_zoneshumides)
0_FONCIER/cutfantoir_bydep.py
+28
View File
@@ -0,0 +1,28 @@
import pandas as pd
import csv
path = '/media/cgeier/SRV/FICHIERS/TRANSFERTS-EQUIPE/CG/FONCIER/CADASTRE/2023/Fichier national FANTOIR (situation avril 2023)/'
fant = path+'FANTOIR0423'
lst_dep = ['07','26','38','42']
if __name__ == '__main__':
df = pd.read_table(fant,chunksize=500000)
d07 = pd.DataFrame()
d26 = pd.DataFrame()
d38 = pd.DataFrame()
d42 = pd.DataFrame()
for d in df:
d.columns=['tmp']
d07 = pd.concat([d07,d[d.tmp.str.startswith('070')]])
d26 = pd.concat([d26,d[d.tmp.str.startswith('260')]])
d38 = pd.concat([d38,d[d.tmp.str.startswith('380')]])
d42 = pd.concat([d42,d[d.tmp.str.startswith('420')]])
d07.to_csv(path+'070.txt', header=None, index=None, quoting=csv.QUOTE_NONE)
d26.to_csv(path+'260.txt', header=None, index=None, quoting=csv.QUOTE_NONE)
d38.to_csv(path+'380.txt', header=None, index=None, quoting=csv.QUOTE_NONE)
d42.to_csv(path+'420.txt', header=None, index=None, quoting=csv.QUOTE_NONE)
0_FONCIER/delete_foncier_acte.py
+15
View File
@@ -0,0 +1,15 @@
from pycen import con_fon
id_acte = 'MU_CHAR_ZI2'
if id_acte.startswith('MF'):
lst_tab = ['r_mfstatut','r_frais_mf','r_cad_site_mf','mf_acquisitions']
key = 'mf_id'
elif id_acte.startswith('MU'):
lst_tab = ['r_mustatut','r_frais_mu','r_cad_site_mu','mu_conventions']
key = 'mu_id'
sql = "DELETE FROM foncier.{tab} WHERE {key}='{acte_id}';"
for t in lst_tab:
with con_fon.begin() as cnx:
cnx.execute(sql.format(tab=t,key=key,acte_id=id_acte))
0_FONCIER/extract_cadastre.py
+102
View File
@@ -0,0 +1,102 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Nom : : extract_cadastre.py
#Description : Extraction du cadastre en fonction du des coordonnées NO,NE,SO,SE
# ou d'un polygon (.shp).
#Copyright : 2021, CEN38
#Auteur : Colas Geier
#Version : 1.0
import pandas as pd
import geopandas as gpd
from sqlalchemy import create_engine
from sqlalchemy.engine import URL
from geoalchemy2 import Geometry
# from shapely.geometry.multipolygon import MultiPolygon
from shapely.geometry.polygon import Polygon
from pyproj import Transformer
from pycen import con_fon as con
# import contextily as ctx
# import matplotlib.pyplot as plt
def geom_map(geom):
map1 = ctx.providers.OpenTopoMap
map2 = ctx.providers.Esri.WorldImagery
maps = geom.to_crs(epsg=3857).copy()
ax = maps.plot(alpha=0.5, edgecolor='k')
ctx.add_basemap(
ax,
attribution_size=5,
reset_extent=False,
source=map1,
zoom='auto'
# zoom=14
)
ax.set_axis_off()
output = '/home/colas/Documents/tmp/AUDE/cadastre_for_Aude.gpkg'
# Shape : chemin/nom.shp -- None
shp = None
# Coordonnées [[NO],[NE],[SE],[SO]] -- None
lat_point_list = [45.414249,45.414611,45.407813,45.407282] #,45.414249]
lon_point_list = [ 5.472428, 5.486879, 5.487019, 5.472701] #, 5.472428]
# bdd
bdd = True
code_site = 'CRAS'
type_zone = None
con_open = con.connect()
# Shape : chemin/nom.shp -- None
# shp = None
# Coordonnées [[NO],[NE],[SE],[SO]] -- None
# lat_point_list = []
# lon_point_list = []
if shp:
print('IMPORT du mask : %s'%shp)
polygon = gpd.read_file(shp)
if polygon.crs.srs != crs.lower():
polygon.to_crs(epsg, inplace=True)
elif bdd:
from pycen import con_bdcen
gdf = gpd.read_postgis("SELECT * FROM sites.c_sites_zonages WHERE code_site = '%s' --and type_zonage = '%s'"%(code_site,type_zone),con_bdcen)
polygon = gdf[['geom']]
polygon.rename_geometry('geometry',inplace=True)
else:
if epsg != 2154:
from pyproj import Transformer
transformer = Transformer.from_crs(crs, "EPSG:2154", always_xy=True)
lon_point_list,lat_point_list = transformer.transform(lon_point_list,lat_point_list)
epsg = 2154
crs = 'EPSG:%s'%epsg
polygon_geom = Polygon(zip(lon_point_list, lat_point_list))
polygon = gpd.GeoDataFrame(index=[0], crs=crs, geometry=[polygon_geom])
sql = '''SELECT t1.par_id, t1.geom, t1.codcom, t4.libelle vl,
t1.ccopre, t1.ccosec, t1.dnupla, t1.dparpi, t5.typprop_lib,t5.typprop, t1.ccocomm,
t1.ccoprem, t1.ccosecm, t1.dnuplam, t1.type,
t8.ccodem,
--substring(t8.dnupro from 3) dnupro,
--substring(t9.dnuper from 3) dnuper,
t9.ddenom, t9.jdatnss, t9.dldnss, t9.dsglpm, t9.dlign3, t9.dlign4,
t9.dlign5, t9.dlign6, t9.dnatpr, t9.gtoper, t9.ccogrm
FROM cadastre.parcelles_38 t1
LEFT JOIN cadastre.vl_38 t4 ON (t1.vl_id = t4.vl_id)
LEFT JOIN cadastre.d_typprop t5 ON (t1.typprop_id = t5.typprop_id)
LEFT JOIN cadastre.lots_38 t2 ON (t1.par_id = t2.par_id)
LEFT JOIN cadastre.lots_natcult_38 t3 ON (t2.lot_id = t3.lot_id)
LEFT JOIN cadastre.cadastre_38 t6 ON (t2.lot_id = t6.lot_id)
LEFT JOIN cadastre.cptprop_38 t7 ON (t6.dnupro = t7.dnupro)
LEFT JOIN cadastre.r_prop_cptprop_38 t8 ON (t7.dnupro = t8.dnupro)
LEFT JOIN cadastre.proprios_38 t9 ON (t8.dnuper = t9.dnuper)
WHERE ST_Intersects (t1.geom, 'SRID={epsg};{poly}');'''.format(epsg=epsg,poly=polygon.geometry[0])
df = gpd.read_postgis(
sql=sql,
con=con,
)
for col in df.columns:
if (df[col].isna()).all():
del df[col]
df.to_file(output,driver='GPKG')
0_FONCIER/foncier_call_fonctions.py
Executable
+137
View File
@@ -0,0 +1,137 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Nom : : foncier_insert_table.py
#Description : Correspondance entre les sites et les parcelles cadastrales.
#Copyright : 2021, CEN38
#Auteur : Colas Geier
#Version : 1.0
import pandas as pd
import geopandas as gpd
from sqlalchemy import create_engine #, func
# from sqlalchemy.orm import sessionmaker
import psycopg2
import datetime as dt
import sys
import gc
from pycen import con_fon
# Parametres généraux
epsg = '2154'
crs = 'EPSG:%s'%epsg
chunk = None
pci_annee = '2021'
matrice_annee = '2021'
start_time = dt.datetime.today()
def time_exec (init_time):
time = dt.datetime.today() - init_time
return str(time)
def get_data(table,schema,engine=con_fon,chunk=chunk):
sql = "SELECT * FROM {0}.{1}".format(schema, table)
tmp = gpd.read_postgis(
sql = sql,
con = con_fon,
geom_col = 'geom',
crs = crs,
chunksize = chunk, )
# Mise en forme des données
# start_time = dt.datetime.today()
if chunk:
df = pd.concat(tmp, ignore_index=True)
else:
df = tmp.copy()
return df
if __name__ == "__main__":
###############
# Get sites
tab_site = 'sites'
sch_site = 'sites'
tab_fon = 'cadastre_site'
sch_fon = 'foncier'
print('''IMPORT data from schema: '%s' , table : '%s' '''%(sch_site,tab_site))
# sql = "SELECT * FROM {0}.{1}".format(sch_site, tab_site)
sql = '''
SELECT * FROM {0}.{1}
WHERE site_id = 'CRAS'
--WHERE site_id NOT IN (SELECT DISTINCT site_id FROM {2}.{3})
'''.format(sch_site, tab_site, sch_fon, tab_fon)
site = gpd.read_postgis(
# table_name = table_in,
sql = sql,
con = con_fon,
geom_col = 'geom',
# schema = schema_in,
crs = crs,
chunksize = chunk, )
# union_site = gpd.GeoSeries(site.geom.cascaded_union)
if site.empty:
print('Pas de nouveaux sites à lier au cadastre ====> EXIT')
sys.exit()
#################
# Get parcelles
tab_parc = 'cadastre'
sch_parc = 'parcelles'
print('''IMPORT data from schema: '%s' , table : '%s' '''%(sch_parc,tab_parc))
sql = """SELECT * FROM {sch}.{tab} WHERE ST_Intersects (geom, 'SRID={epsg};{poly}')
AND par_id NOT IN (SELECT par_id FROM {sch}.parcelles_cen)""".format(
sch=tab_parc, tab=sch_parc, epsg=epsg, poly=site.unary_union
)
parc = gpd.read_postgis(
sql = sql,
con = con_fon,
geom_col = 'geom',
crs = crs,
chunksize = chunk, )
parc_cent = parc.copy()
# parc_cent.geom = parc_cent.representative_point()
# parc_cent.geom = parc_cent.centroid
res = gpd.sjoin(site, parc_cent, predicate='intersects')
print('RUN fonction "import_parcelles_cen" pour {} lignes '.format(res.shape[0]))
# lst_site = res.site_id.unique()
start_time = dt.datetime.today()
res['sql'] = "SELECT cadastre.import_parcelles_cen('"+res.par_id+"','"+res.site_id+"',"+str(start_time.year)+");"
with con_fon.begin() as cnx:
res['sql'].map(lambda x: cnx.execute(x))
###############
# Get cptprop
lst_parid = "','".join(res.par_id.unique())
sql = '''
SELECT * FROM cadastre.cadastre
JOIN cadastre.lots USING (lot_id)
WHERE lots.par_id IN ('{lst_parid}')'''.format(lst_parid=lst_parid)
cptprop = pd.read_sql(
sql = sql,
con = con_fon,)
cptprop.drop_duplicates('dnupro', inplace=True)
start_time = dt.datetime.today()
print('RUN fonction "import_cptprop_cen" pour {} lignes '.format(cptprop.shape[0]))
cptprop['sql'] = "SELECT cadastre.import_cptprop_cen('"+cptprop.dnupro+"',"+str(start_time.year)+");"
with con_fon.begin() as cnx:
cptprop['sql'].map(lambda x: cnx.execute(x))
time_exec(start_time)
print('END fonction : import_cptprop_cen .......... %s'%time_exec(start_time))
site_id = 'GDMA'
with con_fon.begin() as cnx:
cnx.execute("SELECT admin_sig.refresh_mview_foncier('{site_id}')"\
.format(
site_id=site_id
)
)
0_FONCIER/foncier_insert_administratif.py
+437
View File
@@ -0,0 +1,437 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Nom : : foncier_insert_table.py
#Description : Insertion/MAJ des données administratives et territoriales à la base <foncier> lors de sa création.
#Copyright : 2021, CEN38
#Auteur : Colas Geier
#Version : 1.0
import pandas as pd
import geopandas as gpd
from sqlalchemy import create_engine
from geoalchemy2 import Geometry
from pycen import bdd
from shapely.geometry.multipolygon import MultiPolygon
from pydate import cdate
import sys
import os
####################################
####################################
####################################
# PARAMETRES
# Liste des tables à mettre à jour. ATTENTION : le respect des nomenclatures est importante
# Liste dispo : ['com', 'dpt', 'ter', 'histo_com', 'ter_com'] Cette liste doit être identique aux index de l'object : Table
# Correspondance tables BD_FONCIER : ['communes', 'departements', 'territoires', 'r_histo_com', 'r_ter_com']
run = ['com', 'dpt', 'ter', 'histo_com', 'ter_com']
user = 'colas_geier'
pwd = 'adm1n*fOncier'
# Parametres bdd
# user = 'postgres'
# pwd = 'tutu'
# adr = '192.168.60.9'
adr = '91.134.194.221'
port = '5432'
base = 'bd_cen'
# Connexion bdd
# bd = bdd.CEN(
# user = user,
# pwd = pwd,
# adr = adr,
# base = base
# schema = schema
# )
crs = 'EPSG:2154'
con = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user,pwd,adr,port,base), echo=False)
con_open = con.connect()
# create_engine('postgresql+psycopg2://cen_admin:#CEN38@venir@172.17.0.2:5432/bd_cen', echo=True)
# PATH
PATHIN = '/home/colas/'
LIST_DEPT = ['07', '26', '42', '38']
#
MAIN_FOLDER = 'Documents'
SUB_FOLDERS = os.path.join('5_BDD','1_QGIS')
FILE_COM = ['COMMUNE.shp']
FILE_DPT = ['DEPARTEMENT.shp']
FILE_TER = [
'EPCI.shp',
'PARC_OU_RESERVE.shp',
'BASSIN_VERSANT_TOPOGRAPHIQUE.shp',
'Contours_GRPT_AURA.shp' ]
# Dict table
IN_COM = [{
'id': 'id',
'insee_com': 'code_insee',
'nom': 'nom',
None: 'prec_plani',
None: 'statut',
None: 'canton',
'insee_arr': 'arrondisst',
'insee_dep': 'depart',
'insee_reg': 'region',
'population': 'popul',
None: 'multican',
'actif': 'actif',
None: 'epfl',
# 'geometry': 'geom',
}]
IN_DPT = [{
'id': 'id',
'nom': 'nom',
'insee_dep': 'insee_dep',
'insee_reg': 'insee_reg',
'date_creat': 'date_creat',
'date_maj': 'date_maj',
'actif': 'actif',
# 'geometry': 'geom',
}]
IN_TER = [{ # DICT epci
'': 'territoire_id',
'code_siren': 'siren',
'nom': 'territoire_lib',
'': 'territoire_sigle',
'nature': 'typterritoire_id',
'': 'administratif',
'': 'prefixe',
},{ # DICT parc_ou_reserve
'': 'territoire_id',
'id': 'siren', # absence de code siren ==> récup des 10 derniers charactères du champs ID
'toponyme': 'territoire_lib',
'': 'territoire_sigle',
'nature': 'typterritoire_id',
'': 'administratif',
'': 'prefixe',
},{ # DICT bassin_versant
'': 'territoire_id',
'code_hydro': 'siren',
'toponyme': 'territoire_lib',
'': 'territoire_sigle',
'id': 'typterritoire_id',
'': 'administratif',
'': 'prefixe',
},{ # DICT epci
'': 'territoire_id',
'sirengrpt': 'siren',
'grpt': 'territoire_lib',
'': 'territoire_sigle',
'nature': 'typterritoire_id',
'': 'administratif',
'': 'prefixe',
},]
# liste des couches administratives listé dans PATH_TER
administratif = ['epci']
# DICT typterritoire_id lorsqu'une correspondance
# n'est pas possible avec un champs de la table attributaire.
# Relation typterritoire_id / ID (de la table attributaire).
typterid = {
'bassvers' : 'bv'
}
mask_path = 'Documents/5_BDD/1_QGIS/'
mask_file = 'mask_parcelles_cadastre.shp'
####################################
####################################
####################################
# FONCTIONS
def join_typterritoire(df, join):
tab = join['table']
sch = join['schema']
ind = join['id']
on = join['on']
tmp = pd.read_sql_table(
table_name = tab,
con = con,
schema = sch,
index_col = ind,
)
df[on['x']] = df[on['x']].str.lower()
df[on['x']] = df[on['x']].replace(tmp[on['y']].str.lower().to_list(),tmp.index.to_list())
df = df[ df[on['x']].isin(tmp.index.to_list()) ]
return df
def find_files(File, main_path='Documents', sub_path=None):
'''
@File : list
@main_path : str
@sub_path : str
'''
sub = ''
if sub_path:
sub = sub_path
path = os.path.join(main_path, sub)
list_path = []
for F in File :
matches = [str(path) for path in Path(path).rglob(F)]
list_path += matches
return list_path
####################################
####################################
####################################
# MAIN
if os.path.exists(PATHIN + mask_path + mask_file):
mask = gpd.read_file(PATHIN + mask_path + mask_file, crs=crs)
else :
sys.exit()
Table = {
'dpt': {'schema':'administratif', 'table':'departements', 'file': FILE_DPT, 'geom': True, 'dict':IN_DPT},
'com' : {'schema':'administratif', 'table':'communes', 'file': FILE_COM, 'geom': True, 'dict':IN_COM},
# 'histo_com': {'schema':'administratif', 'table':'r_histo_com', 'file':None, 'geom': False, 'dict':None},
'ter': {'schema':'territoires', 'table':'territoires', 'file': FILE_TER, 'geom': True, 'dict':IN_TER, 'join': {
'schema':'territoires', 'table': 'd_typterritoire', 'id': 'typterritoire_id', 'on': {'x': 'typterritoire_id', 'y': 'typterritoire_lib'}}},
'ter_com': {'schema':'territoires', 'table':'r_ter_com', 'file':None, 'geom': False, 'dict':None}
}
for d, dep in enumerate(LIST_DEPT):
PATH = 'Documents/5_BDD/1_QGIS/202103_BDTOPO/bdtopo_dep{0}_202103_shapefile/BDT_3-0_SHP_LAMB93_D0{0}-ED2021-03-15/'.format(dep)
PATH_COM = [PATH + 'ADMINISTRATIF/COMMUNE.shp']
PATH_DPT = [PATH + 'ADMINISTRATIF/DEPARTEMENT.shp']
PATH_TER = [
PATH + 'ADMINISTRATIF/EPCI.shp',
PATH + 'ZONES_REGLEMENTEES/PARC_OU_RESERVE.shp',
PATH + 'HYDROGRAPHIE/BASSIN_VERSANT_TOPOGRAPHIQUE.shp',
'' ]
Table = {
'dpt': {'schema':'administratif', 'table':'departements', 'file': [PATHIN + path for path in PATH_DPT], 'geom': True, 'dict':IN_DPT},
'com' : {'schema':'administratif', 'table':'communes', 'file': [PATHIN + path for path in PATH_COM], 'geom': True, 'dict':IN_COM},
# 'histo_com': {'schema':'administratif', 'table':'r_histo_com', 'file':None, 'geom': False, 'dict':None},
'ter': {'schema':'territoires', 'table':'territoires', 'file': [PATHIN + path for path in PATH_TER], 'geom': True, 'dict':IN_TER, 'join': {
'schema':'territoires', 'table': 'd_typterritoire', 'id': 'typterritoire_id', 'on': {'x': 'typterritoire_id', 'y': 'typterritoire_lib'}}},
'ter_com': {'schema':'territoires', 'table':'r_ter_com', 'file':None, 'geom': False, 'dict':None}
}
not_run = [k for k in Table.keys() if k not in run]
for r in not_run:
del Table[r]
for tab in Table:
Table['file'] = find_files(File= Table['file'], sub_path=SUB_FOLDERS, main_path=MAIN_FOLDER)
if d == 0:
for tab in reversed(Table):
# continue
sql = "TRUNCATE TABLE {0}.{1} CASCADE".format(Table[tab]['schema'], Table[tab]['table'])
print(sql)
con_open.execute(sql)
no_r_tab = lambda x: x not in ['ter_com']
for key in filter(no_r_tab, Table.keys()):
# Test existance de la table en bdd
lst_tab = con.dialect.get_table_names(con, schema=Table[key]['schema'])
test = Table[key]['table'] in lst_tab
# Si la table existe
if test:
DICT = Table[key]['dict']
# Test présence d'un champ 'geom' ou 'geometry' dans la table d'export
geom = False
col_tab = con.dialect.get_columns(con, Table[key]['table'], schema=Table[key]['schema'])
for o, obj in enumerate(col_tab):
if 'geom' in obj['name']:
geom = True
geom_name = obj['name']
geom_type = obj['type'].geometry_type
if DICT:
for D, tmp in enumerate(DICT):
DICT[D]['geometry'] = geom_name
# Suppression des champs non utiles
if DICT:
for D, tmp in enumerate(DICT):
if DICT[D] and None in DICT[D].keys():
del DICT[D][None]
if Table[key]['file']:
for f, i_file in enumerate(Table[key]['file']):
# Si présence d'une géometrie dans la table à insérer
if geom:
# if Table[key]['geom']:
# Read new table
print('IMPORT shape for table {0}'.format(Table[key]['table']))
df = gpd.read_file(filename=i_file)
df = gpd.sjoin(df, mask, how='inner', op='intersects', rsuffix='right')
del_cols = [col for col in tutu.columns if col.endswith('right')] + ['FID']
df.drop(columns=del_cols, inplace=True)
df['actif'] = True
# if 'ID' in df.columns:
# df.set_index('ID', inplace=True)
# typ_geom_out = con.dialect.get_columns(con, Table[key]['table'], schema=Table[key]['schema'])
# Harmonisation des géometries
# Transformation des géometries POLYGON ==> MULTIPOLIGON
geom_df = df.geometry.geom_type.unique().tolist()
geom_df = [x.upper() for x in geom_df]
if [geom_type] != geom_df:
if geom_type == 'MULTIPOLYGON' and 'POLYGON' in geom_df:
print('CORRECTION des géometries POLYGON ==> MULTIPOLYGON')
lst = []
for o, obj in enumerate(df.geometry):
if obj.geom_type == 'Polygon':
obj = MultiPolygon([obj])
lst.append(obj)
df['geometry'] = lst
# elif geom_type == 'POLYGON' and 'MULTIPOLYGON' in geom_df:
# df[df.geom.geom_type == 'MultiPolygon']
# pass
else:
print('ERROR : conflit entre la géometrie du df {0} et de la table postgis {1}'.format(geom_df,geom_type))
# Conservation des lignes appartenant au département
print('CONSERVATION des données départementales')
df.columns = df.columns.str.lower()
if 'insee_dep' in df.columns:
df = df.loc[df.insee_dep == dep]
# Formatage des champs pour insertion
print('FORMATAGE des données')
df.rename(columns=DICT[f], inplace=True)
rm_col = df.columns.difference(DICT[f].values())
df.drop(columns=rm_col, inplace=True)
# break
# Identification du champs 'administratif' pour la table territoire
couche = i_file.split('/')
couche = couche[len(couche)-1]
couche = couche.split('.')[0].lower()
if 'ter' == key and couche in administratif:
df['administratif'] = True
elif 'ter' == key and couche not in administratif:
df['administratif'] = False
df['siren'] = [siren[-10:] for siren in df['siren']]
if 'typterritoire_id' in DICT[f].values():
key_typterr = [k for (k, v) in DICT[f].items() if v == 'typterritoire_id'][0]
if 'join' in Table[key].keys() and key_typterr != 'id':
df = join_typterritoire(df, Table[key]['join'])
if key == 'ter' and key_typterr == 'id':
# df['typterritoire_id'] = df.index.to_list()
df['typterritoire_id'] = [typter[:8] for typter in df['typterritoire_id']]
df['typterritoire_id'] = df['typterritoire_id'].str.lower()
df['typterritoire_id'] = df['typterritoire_id'].replace(typterid)
# df[on['x']].replace(tmp[on['y']].str.lower().to_list(),tmp.index.to_list())
# Si présence d'une géometrie dans la table à insérer
if geom:
if not isinstance(df, gpd.GeoDataFrame):
df = df.set_geometry('geom', drop=True, crs=crs)
df.rename(columns={'geometry': 'geom'}, inplace=True)
# if Table[key]['geom']:
df.to_postgis(
name = Table[key]['table'],
con = con,
schema = Table[key]['schema'],
index = False,
if_exists = 'append',
geom_col = geom_name,
# dtype={'geom': Geometry(geometry_type='MULTIPOLYGON', srid=df.crs.to_epsg())}
)
print('''INSERT TABLE OK for DEPT {}
'''.format(dep))
else:
None
# Si la table existe pas
else:
print('ERROR : La table {0} n\'existe pas dans le schéma {1} !'.format(
Table[key]['table'].upper(),
Table[key]['schema'].upper()
))
if 'ter_com' in Table.keys() and d == 0:
tab = Table['ter_com']
print('IMPORT tables for table {0}'.format(tab['table']))
ter_sql = 'SELECT * FROM {sch}.{tab}'.format(sch='territoires', tab='territoires' )
ter = gpd.read_postgis(
sql = ter_sql,
con = con,
geom_col = 'geom',
crs = crs,
)
com_sql = 'SELECT * FROM {sch}.{tab} WHERE actif = true'.format(sch='administratif', tab='communes' )
com = gpd.read_postgis(
sql = com_sql,
con = con,
geom_col = 'geom',
crs = crs,
)
col_id = ['territoire_id', 'code_insee', 'geom']
for df in [ter, com]:
rm_col = [ col for col in df.columns[~df.columns.isin(col_id)] ]
df.drop(columns=rm_col, inplace=True)
print('JOIN tables "territoires" & "communes"')
df = gpd.sjoin(ter, com, op='intersects')
rm_col = [ col for col in df.columns[~df.columns.isin(col_id)] ]
rm_col.append('geom')
df.drop(columns=rm_col, inplace=True)
df = pd.DataFrame(df)
r_sql = 'SELECT code_insee, territoire_id FROM {sch}.{tab}'.format(sch=tab['schema'], tab=tab['table'] )
r_tab = pd.read_sql(
sql = r_sql,
con = con,
)
if not r_tab.empty:
print('DROP lignes présentes dans la table {}'.format(tab['table']))
df = pd.concat([df,r_tab]).drop_duplicates(keep=False)
if not df.empty:
users = pd.read_sql_table(
table_name = 'utilisateurs',
con = con,
schema = 'admin_sig'
)
date_now = cdate.today()
df['actif'] = True
df['date_maj'] = date_now
df['utilisateur_id'] = users[users.utilisateur_id == user].iloc[0]['individu_id']
df.to_sql(
name = tab['table'],
con = con,
schema = tab['schema'],
index = False,
if_exists = 'append'
)
else:
print('''
TOUTES les relations "communes" / "territoires" existent déjà !
''')
# gdf.set_index('id', drop=True, inplace=True)
# for key in Table.keys():
# # schema = Table[key]['schema']
# # table = Table[key]['table']
# # query = 'SELECT * FROM {0}.{1}'.format(schema,table)
# # gdf = gpd.read_postgis(sql=query, con=con)
# df = bd.get_table(
# schema=Table[key]['schema'],
# table=Table[key]['table'])
# print(df)
0_FONCIER/foncier_insert_administratif_V2.py
Executable
+573
View File
@@ -0,0 +1,573 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Nom : : foncier_insert_table.py
#Description : Insertion/MAJ des données administratives et territoriales à la base <foncier> lors de sa création.
#Copyright : Mai 2021, CEN38
#Auteur : Colas Geier
#Version : 2.0
import pandas as pd
import geopandas as gpd
from sqlalchemy import create_engine, text
from geoalchemy2 import Geometry
from shapely.geometry.multipolygon import MultiPolygon
from pydate import cdate
from pathlib import Path
from pycen import con_fon as con
import sys
import os
# from pycen import bdd
# import psycopg2
####################################
####################################
####################################
# PARAMETRES
# Liste des tables à mettre à jour. ATTENTION : le respect des nomenclatures est importante
# Liste dispo : ['com', 'dpt', 'ter', 'ter_com'] Cette liste doit être identique aux index de l'object : Table
# Correspondance tables BD_FONCIER : ['communes', 'departements', 'territoires', 'r_histo_com', 'r_ter_com']
run = ['com']
# INSERT, TRUNCATE, UPDATE
action = 'UPDATE'
# Parametres bdd
# user = 'remi_clement'
# pwd = 'adm1n*CEN'
# adr = '192.168.60.9'
user = 'colas_geier'
pwd = 'adm1n*fOncier'
adr = '91.134.194.221'
port = '5432'
base = 'bd_cen'
epsg = '2154'
crs = 'EPSG:%s'%epsg
# PATH
INDEX_FOLDER = '/home/colas' # "/home/USER" (Unix), "C:" ou autre (Windows)
MAIN_FOLDER = os.path.join(INDEX_FOLDER,'Documents') # Dossier principale
SUB_FOLDERS = os.path.join('5_BDD','1_QGIS') # Sous dossier (facultatif: os.path.join('subFolder1','subFolder2'), None si non précisé)
FILE_COM = ['COMMUNE.shp']
FILE_DPT = ['DEPARTEMENT.shp']
FILE_TER = [
'EPCI.shp',
'PARC_OU_RESERVE.shp',
'BASSIN_VERSANT_TOPOGRAPHIQUE.shp',
'Contours_GRPT_AURA.shp' ]
administratif = ['epci'] # liste des couches administratives listé dans PATH_TER
mask_Mfolder = os.path.join(INDEX_FOLDER,'Documents')
mask_Sfolder = os.path.join('5_BDD','1_QGIS')
mask_file = 'mask_parcelles_cadastre.shp'
# DICT FILE / TABLE
IN_COM = [{
'id': 'id',
'insee_com': 'code_insee',
'nom': 'nom',
None: 'prec_plani', # Plus fournis par la bd_topo
None: 'statut', # Plus fournis par la bd_topo
None: 'canton', # Plus fournis par la bd_topo
'insee_arr': 'arrondisst',
'insee_dep': 'depart',
'insee_reg': 'region',
'population': 'popul',
None: 'multican', # Plus fournis par la bd_topo
'actif': 'actif',
None: 'epfl', # Plus fournis par la bd_topo
}]
IN_DPT = [{
'id': 'id',
'nom': 'nom',
'insee_dep': 'insee_dep',
'insee_reg': 'insee_reg',
'date_creat': 'date_creat',
'date_maj': 'date_maj',
'actif': 'actif',
}]
IN_TER = [{ # DICT epci
'': 'territoire_id',
'code_siren': 'siren',
'nom': 'territoire_lib',
'': 'territoire_sigle',
'nature': 'typterritoire_id',
'': 'administratif',
'': 'prefixe',
},{ # DICT parc_ou_reserve
'': 'territoire_id',
'id': 'siren', # absence de code siren ==> récup des 10 derniers charactères du champs ID
'toponyme': 'territoire_lib',
'': 'territoire_sigle',
'nature': 'typterritoire_id',
'': 'administratif',
'': 'prefixe',
},{ # DICT bassin_versant
'': 'territoire_id',
'code_hydro': 'siren', # absence de code siren ==> récup des 10 derniers charactères du champs ID
'toponyme': 'territoire_lib',
'': 'territoire_sigle',
'id': 'typterritoire_id',
'': 'administratif',
'': 'prefixe',
},{ # DICT epci
'': 'territoire_id',
'sirengrpt': 'siren',
'grpt': 'territoire_lib',
'': 'territoire_sigle',
'nature': 'typterritoire_id',
'': 'administratif',
'': 'prefixe',
},]
# DICT typterritoire_id lorsqu'une correspondance
# n'est pas possible avec un champs de la table attributaire.
# Relation typterritoire_id / ID (de la table attributaire).
typterid = {
'bassvers' : 'bv'
}
Table = {
'dpt': {'schema':'administratif', 'table':'departements', 'name_file': FILE_DPT, 'geom': True, 'dict':IN_DPT, 'unique': 'insee_dep'},
'com' : {'schema':'administratif', 'table':'communes', 'name_file': FILE_COM, 'geom': True, 'dict':IN_COM, 'unique': 'code_insee'},
# 'histo_com': {'schema':'administratif', 'table':'r_histo_com', 'name_file':None, 'geom': False, 'dict':None},
'ter': {'schema':'territoires', 'table':'territoires', 'name_file': FILE_TER, 'geom': True, 'dict':IN_TER, 'unique': 'siren', 'join': {
'schema':'territoires', 'table': 'd_typterritoire', 'id': 'typterritoire_id', 'on': {'x': 'typterritoire_id', 'y': 'typterritoire_lib'}}},
'ter_com': {'schema':'territoires', 'table':'r_ter_com', 'name_file':None, 'geom': False, 'dict':None, 'unique': ['code_insee', 'territoire_id']}
}
####################################
####################################
####################################
# FONCTIONS
def join_typterritoire(df, join):
'''
@df : dataframe
@join : dict
'''
tab = join['table']
sch = join['schema']
ind = join['id']
on = join['on']
tmp = pd.read_sql_table(
table_name = tab,
con = con,
schema = sch,
index_col = ind,
)
df[on['x']] = df[on['x']].str.lower()
df[on['x']] = df[on['x']].replace(tmp[on['y']].str.lower().to_list(),tmp.index.to_list())
df = df[ df[on['x']].isin(tmp.index.to_list()) ]
return df
def find_files(File, main_path='Documents', sub_path=None):
'''
@File : list
@main_path : str
@sub_path : str
'''
sub = ''
if sub_path:
sub = sub_path
path = os.path.join(main_path, sub)
list_path = []
nb_path = []
for F in File :
matches = [str(path) for path in Path(path).rglob(F)]
list_path += matches
nb_path += str(len(matches))
return list_path, nb_path
def tab_has_data(con, schema, table):
'''
@con : connection sqlalchemy create_engine
@schema : str
@table : str
'''
has_sch = con.dialect.has_schema(con, schema=schema)
if has_sch :
has_table = con.dialect.has_table(con, table_name=table, schema=schema)
if has_table:
sql = 'SELECT * FROM {sch}.{tab} LIMIT 1'.format(sch=schema,tab=table)
df = pd.read_sql_query(
sql = sql,
con = con
)
return not df.empty
else:
return '''TABLE %s don't exist in SCHEMA %s''' %(table,schema)
else :
return '''SCHEMA %s don't exist'''%schema
def update_data(df, con, sch, tab, epsg=None):
columns = df.columns.to_list()
frame = df.copy()
frame.replace("'","''", regex=True, inplace=True)
pkey = con.dialect.get_pk_constraint(con, table_name=tab, schema=sch)['constrained_columns']
if 'geom' in columns or 'geometry' in columns:
if epsg or df.crs:
if not epsg:
epsg = df.crs.to_epsg()
name_geom = df.geometry.name
frame[name_geom] = 'SRID={epsg};'.format(epsg=epsg) + df[name_geom].map(str)
# else: return 'No crs define in update_data or in gdf'
for c, col in enumerate(columns):
if c == 0:
frame['insert'] = "('" + frame[col].map(str)
# break
else:
frame['insert'] = frame['insert'] + "','" + frame[col].map(str)
if c == len(columns)-1:
frame['insert'] = frame['insert'] + "')"
# if c == 0:
# frame['insert'] = '("' + frame[col].map(str)
# # break
# else:
# frame['insert'] = frame['insert'] + '","' + frame[col].map(str)
# if c == len(columns)-1:
# frame['insert'] = frame['insert'] + '")'
lst_cols = ', '.join(columns)
lst_vals = ','.join(frame['insert'])
lst_dupKey = ', '.join([col + '=EXCLUDED.' + col for col in columns])
lst_pkey = ','.join(pkey)
sql = '''INSERT INTO {sch}.{tab} ({lst_cols}) VALUES {lst_vals} ON CONFLICT ({lst_pkey}) DO UPDATE SET {lst_dupKey} ;'''.format(
sch=sch, tab=tab, lst_cols=lst_cols, lst_vals=lst_vals, lst_dupKey=lst_dupKey, lst_pkey=lst_pkey)
# sql = '''INSERT INTO {sch}.{tab} ({lst_cols})
# VALUES {lst_vals}
# ON CONFLICT DO NOTHING;
# '''.format(sch=sch, tab=tab, lst_cols=lst_cols, lst_vals=lst_vals)
try:
con.execute(sql)
# con.execute(text(sql))
print('''
Update OK !''')
except Exception as exept:
print(exept)
# con = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user,pwd,adr,port,base), echo=False)
con_open = con.connect()
# conn = psycopg2.connect(
# user=user,
# password=pwd,
# host=adr,
# port=port,
# database=base)
####################################
####################################
####################################
# MAIN
if __name__ == "__main__":
mask_f = find_files(File=[mask_file], sub_path=mask_Sfolder, main_path=mask_Mfolder)[0]
if mask_f:
mask = gpd.read_file(mask_f[0], crs=crs)
else :
sys.exit('NO MASK FILE')
not_run = [k for k in Table.keys() if k not in run]
for r in not_run:
del Table[r]
for tab in Table :
if Table[tab]['name_file']:
Table[tab]['file'], Table[tab]['nb_typ_file'] = find_files(File=Table[tab]['name_file'], sub_path=SUB_FOLDERS, main_path=MAIN_FOLDER)
d = []
for i, nb in enumerate(Table[tab]['nb_typ_file']):
d += [ Table[tab]['dict'][i] ] * int(nb)
Table[tab]['dict'] = d
else:
Table[tab]['file'] = None
if action == 'TRUNCATE':
for tab in reversed(Table):
# continue
sql = "TRUNCATE TABLE {0}.{1} CASCADE".format(Table[tab]['schema'], Table[tab]['table'])
print(sql)
con_open.execute(sql)
sys.exit('END TRUNCATE')
else:
# filter1 = lambda x: x not in ['ter_com']
# for key in filter(filter1, Table.keys()):
for key in Table.keys():
# Test existance de données dans la table en bdd
has_data = tab_has_data(con, schema=Table[key]['schema'], table=Table[key]['table'])
# Si la table existe
if type(has_data) == bool :
# récupération des données présentent en bdd
if type(Table[key]['unique']) == str:
Table[key]['unique'] = [ Table[key]['unique'] ]
df_exist = False
if has_data :
df_exist = pd.read_sql_table(
con = con,
schema = Table[key]['schema'],
table_name = Table[key]['table'],
columns = Table[key]['unique']
)
DICT = Table[key]['dict']
# Test présence d'un champ 'geom' ou 'geometry' dans la table d'export
# Si géometrie, ajout du champs au dictionnaire
geom = False
col_tab = con.dialect.get_columns(con, Table[key]['table'], schema=Table[key]['schema'])
for o, obj in enumerate(col_tab):
if 'geom' in obj['name']:
geom = True
geom_name = obj['name']
geom_type = obj['type'].geometry_type
if DICT:
for D, tmp in enumerate(DICT):
DICT[D]['geometry'] = geom_name
# Récupération des données existantes dans la base de données
# Suppression des champs non utiles
if DICT:
for D, tmp in enumerate(DICT):
if DICT[D] and None in DICT[D].keys():
del DICT[D][None]
if Table[key]['file']:
for f, i_file in enumerate(Table[key]['file']):
# Test existance de données dans la table en bdd
has_data = tab_has_data(con, schema=Table[key]['schema'], table=Table[key]['table'])
# Si la table existe
if type(has_data) == bool :
# récupération des données présentent en bdd
if type(Table[key]['unique']) == str:
Table[key]['unique'] = [ Table[key]['unique'] ]
df_exist = False
if has_data :
pkey = con.dialect.get_pk_constraint(con, table_name=Table[key]['table'], schema=Table[key]['schema'])['constrained_columns']
df_exist = pd.read_sql_table(
con = con,
schema = Table[key]['schema'],
table_name = Table[key]['table'],
columns = Table[key]['unique'],
index_col = pkey
)
if df_exist.shape[1] == 0 :
df_exist[Table[key]['unique']] = df_exist.index
print('''
IMPORT {1} for table {0}'''.format(Table[key]['table'], i_file))
# Si présence d'une géometrie dans la table à insérer
if geom:
# if Table[key]['geom']:
# Read new table
df = gpd.read_file(filename=i_file)
if not df.crs:
df.set_crs(crs=crs, inplace=True)
df = gpd.sjoin(df, mask, how='inner', op='intersects', rsuffix='right')
del_cols = [col for col in df.columns if col.endswith('right')] + ['FID']
df.drop(columns=del_cols, inplace=True)
df['actif'] = True
geom_df = df.geometry.geom_type.unique().tolist()
geom_df = [x.upper() for x in geom_df]
if [geom_type] != geom_df:
if geom_type == 'MULTIPOLYGON' and 'POLYGON' in geom_df:
print('CORRECTION des géometries POLYGON ==> MULTIPOLYGON')
lst = []
for o, obj in enumerate(df.geometry):
if obj.geom_type == 'Polygon':
obj = MultiPolygon([obj])
lst.append(obj)
df['geometry'] = lst
# elif geom_type == 'POLYGON' and 'MULTIPOLYGON' in geom_df:
# df[df.geom.geom_type == 'MultiPolygon']
# pass
else:
print('ERROR : conflit entre la géometrie du df {0} et de la table postgis {1}'.format(geom_df,geom_type))
else:
# Si le fichier à importer ne possède pas de géometrie
print('NO geom !')
print('IMPORT data without geom : No-config !')
# Conservation des lignes appartenant au département
print('CONSERVATION des données départementales')
df.columns = df.columns.str.lower()
# if 'insee_dep' in df.columns:
# df = df.loc[df.insee_dep == dep]
# Formatage des champs pour insertion
print('FORMATAGE des données')
df.rename(columns=DICT[f], inplace=True)
rm_col = df.columns.difference(DICT[f].values())
df.drop(columns=rm_col, inplace=True)
# Identification du champs 'administratif' pour la table territoire
couche = i_file.split('/')
couche = couche[len(couche)-1]
couche = couche.split('.')[0].lower()
if 'ter' == key and couche in administratif:
df['administratif'] = True
elif 'ter' == key and couche not in administratif:
df['administratif'] = False
if df['siren'].dtypes == float:
df['siren'] = df['siren'].astype(int)
if df['siren'].dtypes == int:
df['siren'] = df['siren'].astype(str)
df['siren'] = [siren[-10:] for siren in df['siren']]
if 'typterritoire_id' in DICT[f].values():
key_typterr = [k for (k, v) in DICT[f].items() if v == 'typterritoire_id'][0]
if 'join' in Table[key].keys() and key_typterr != 'id':
df = join_typterritoire(df, Table[key]['join'])
if key == 'ter' and key_typterr == 'id':
df['typterritoire_id'] = [typter[:8] for typter in df['typterritoire_id']]
df['typterritoire_id'] = df['typterritoire_id'].str.lower()
df['typterritoire_id'] = df['typterritoire_id'].replace(typterid)
# if key == 'com':
# sys.exit()
# Suppression des lignes existantes en bdd
if action == 'INSERT' and has_data:
unique_cols = df_exist.columns.to_list()
for d in [df, df_exist]:
d['exist'] = ''
for col in unique_cols:
d['exist'] += d[col].astype(str)
df = df[~df['exist'].isin(df_exist['exist'])]
df.drop(columns='exist', inplace=True)
if action == 'UPDATE':
unique_cols = df_exist.columns.to_list()
for d in [df, df_exist]:
d['exist'] = ''
for col in unique_cols:
d['exist'] += d[col].astype(str)
df = df[df['exist'].isin(df_exist['exist'])].sort_values(unique_cols)
ind = df_exist[df_exist['exist'].isin(df['exist'])].sort_values(unique_cols).index
df.set_index(ind, inplace=True)
df.drop(columns='exist', inplace=True)
# Si présence d'une géometrie dans la table à insérer
if geom and not df.empty and action == 'INSERT' :
if not isinstance(df, gpd.GeoDataFrame):
df = df.set_geometry('geom', drop=False, crs=crs)
# df.rename(columns={'geometry': 'geom'}, inplace=True)
df.to_postgis(
name = Table[key]['table'],
con = con,
schema = Table[key]['schema'],
index = False,
if_exists = 'append',
geom_col = geom_name,
)
elif geom and df.empty and action == 'INSERT' :
print('NO NEWS data insert !')
# Si présence d'une géometrie dans la table à updater
elif geom and not df.empty and action == 'UPDATE' :
if not isinstance(df, gpd.GeoDataFrame):
df = df.set_geometry('geom', drop=False, crs=crs)
df.reset_index(inplace=True)
update_data(df, con, sch=Table[key]['schema'], tab=Table[key]['table'])
print('NO NEWS data update !')
sys.exit()
elif geom and df.empty and action == 'UPDATE' :
print('NO NEWS data update !')
else:
# Si les données à importer sont issues d'un fichier sans géometrie
print('FILE WITHOUT GEOM !')
else:
# Si il n'y a pas de données à importer
# Création des liens relationnelles Communes/Territoires.
print('NO IMPORT FILE !')
tab = Table[key]
print('IMPORT tables for table {0}'.format(tab['table']))
# SELECT data territoires
ter_sql = 'SELECT * FROM {sch}.{tab}'.format(sch='territoires', tab='territoires' )
ter = gpd.read_postgis(
sql = ter_sql,
con = con,
geom_col = 'geom',
crs = crs,
)
# SELECT DATA communes
com_sql = 'SELECT * FROM {sch}.{tab} WHERE actif = true'.format(sch='administratif', tab='communes' )
com = gpd.read_postgis(
sql = com_sql,
con = con,
geom_col = 'geom',
crs = crs,
)
# Conservation des données utiles
tab['unique'] += ['geom']
for df in [ter, com]:
rm_col = [ col for col in df.columns[~df.columns.isin(tab['unique'])] ]
df.drop(columns=rm_col, inplace=True)
# Jointure territoires VS communes
if not ter.empty and not com.empty:
print('JOIN tables "territoires" & "communes"')
df = gpd.sjoin(ter, com, op='intersects')
rm_col = [ col for col in df.columns[~df.columns.isin(tab['unique'])] ]
rm_col.append('geom')
df.drop(columns=rm_col, inplace=True)
df = pd.DataFrame(df)
# Récupération des données déjà présentent en bdd
# r_sql = 'SELECT code_insee, territoire_id FROM {sch}.{tab}'.format(sch=tab['schema'], tab=tab['table'] )
# r_tab = pd.read_sql(
# sql = r_sql,
# con = con,
# )
# if not r_tab.empty:
# print('DROP lignes présentes dans la table {}'.format(tab['table']))
# df = pd.concat([df,r_tab]).drop_duplicates(keep=False)
if action == 'INSERT' and has_data:
df = pd.concat([df,df_exist]).drop_duplicates(keep=False)
if action == 'INSERT' and not df.empty:
users = pd.read_sql_table(
table_name = 'utilisateurs',
con = con,
schema = 'admin_sig'
)
date_now = cdate.today()
df['actif'] = True
df['date_maj'] = date_now
df['utilisateur_id'] = users[users.utilisateur_id == user].iloc[0]['individu_id']
df.to_sql(
name = tab['table'],
con = con,
schema = tab['schema'],
index = False,
if_exists = 'append'
)
else:
print('''
TOUTES les relations "communes" / "territoires" existent déjà !
''')
elif ter.empty and not com.empty:
print('''
NO DATA dans la table "territoires" !
''')
elif not ter.empty and com.empty:
print('''
NO DATA dans la table "communes" !
''')
else:
print('''
NO DATA dans la table "communes" et dans la table "territoires" !
''')
sys.exit()
0_FONCIER/foncier_insert_cadastre.py
Executable
+770
View File
@@ -0,0 +1,770 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Nom : : foncier_insert_table.py
#Description : Insertion des données cadastrales à la base <foncier> après de sa création.
#Copyright : 2021, CEN38
#Auteur : Colas Geier
#Version : 1.0
import pandas as pd
# import numpy as np
from sqlalchemy import create_engine, text
from geoalchemy2 import Geometry
import gc
import sys
# import time
import datetime as dt
# from pycen import bdd
# from shapely.geometry.multipolygon import MultiPolygon
# Parametrage geopandas
import geopandas as gpd
import warnings; warnings.filterwarnings('ignore', 'GeoSeries.isna', UserWarning)
# import shapely
# shapely.speedups.disable()
# gpd.options.use_pygeos = True
# start_time = dt.datetime.today()
# tmp = dt.datetime.today() - start_time
check_duplicates = False
# Parametres bdd CADASTRE (in)
# Données de sortie du plugin qgis "Cadastre"
user_cad = 'postgres'
pwd_cad = 'foncier_test1'
adr_cad = '172.17.0.2'
port_cad = '5432'
base_cad = 'postgres'
schema_cad = '202007'
# Parametres bdd FONCIER (out)
user_fon = 'postgres'
pwd_fon = 'tutu'
adr_fon = '192.168.60.9'
port_fon = '5432'
base_fon = 'bd_cen'
schema_fon = 'cadastre'
# Correspondance entre les tables
crs = 'EPSG:2154'
dpt_nom_tab = '_73'
chunk = 100000
list_dep = ['07', '26', '42', '38']
FIND_DOUBLON = [{
'tab_in': 'proprietaire',
'on_col': ['ddenom', 'dprnlp', 'dldnss','jdatnss','ccogrm','dsglpm','dnatpr'] }
]
DICT_TAB = [{
'table_in' : 'proprietaire', # Table source qui provient de la sortie du plugin cadastre de qgis
'index_tab': 'proprietaire', # Pkey de la table source
'columns_in': ['ccodep', 'ccocom', 'dnupro',
'dnuper', 'ccoqua', 'ddenom', 'jdatnss', 'dldnss', 'dsglpm', 'dlign3', 'dlign4', 'dlign5', 'dlign6', 'dnatpr', 'gtoper', 'ccogrm'],
'table_out': [{
'name': 'cptprop{}'.format(dpt_nom_tab),
'geom': None,
'drop_escape': False, # Supprime les champs vides à l'intérieure des chaines de carractères
'columns_in': ['ccodep', 'ccocom', 'dnupro'], # Liste des columns à récupérer en entrée.
'columns_add': {'dnupro': ['ccodep', 'ccocom', 'dnupro']}, # Définition des champs composés devant être ajoutés
'unique': {'cols': ['dnupro'], 'keep': 'first'}, # Champs devant être uniques à l'intérieur de la table en sortie
'dict': None, # Dictionnaire pour renommer les champs {'ancien_nom1': 'nouveau_nom1', 'ancien_nom2': 'nouveau_nom2', ...}
'join': [{
'bdd': 'in', 'table': 'suf', 'on': ['ccodep', 'ccocom', 'dnupro'], 'type': 'concat',
'select_cols' : ['ccodep', 'ccocom', 'dnupro']},{
'bdd': 'in', 'table': 'lots', 'on': ['ccodep', 'ccocom', 'dnupro'], 'type': 'concat',
'select_cols' : ['ccodep', 'ccocom', 'dnuprol'],'dict': {'dnuprol': 'dnupro'}},{
'bdd': 'in', 'table': 'parcelle', 'on': ['ccodep', 'ccocom', 'dnupro'], 'type': 'concat',
'select_cols' : ['ccodep', 'ccocom', 'dnupro']},]
},{
'name': 'proprios{}'.format(dpt_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in': ['ccodep', 'dnuper', 'ccoqua', 'ddenom', 'jdatnss', 'dldnss', 'dsglpm', 'dlign3', 'dlign4', 'dlign5', 'dlign6', 'dnatpr', 'gtoper', 'ccogrm'],
'columns_add': {'dnuper': ['ccodep', 'dnuper']},
'unique': {'cols': ['dnuper'], 'keep': 'first'},
'dict': None,
'join': False
},{
'name': 'r_prop_cptprop{}'.format(dpt_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in': ['ccodep', 'dnuper', 'ccocom', 'dnupro', 'dnomlp', 'dprnlp', 'epxnee', 'dnomcp', 'dprncp', 'ccodro', 'ccodem'],
'columns_add': {
'dnuper': ['ccodep', 'dnuper'],
'dnupro': ['ccodep', 'ccocom', 'dnupro']},
'unique': {'cols': ['dnupro', 'dnuper'], 'keep': 'first'},
'dict': None,
'join': False
},]
},{
'table_in' : 'parcelle',
'index_tab': 'parcelle',
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'ccovoi', 'dparpi', 'dcntpa', 'ccocomm', 'ccoprem', 'ccosecm', 'dnuplam', 'dvoilib', 'type_filiation', 'dnupro'],
'table_out': [{
'name': 'vl{}'.format(dpt_nom_tab),
'geom': None,
'drop_escape': False,
'columns_in' : ['ccodep', 'ccocom', 'ccovoi', 'dvoilib'],
'columns_add': {
'vl_id': ['ccodep', 'ccocom', 'ccovoi'],
'geom': None},
'unique': {'cols': ['vl_id'], 'keep': 'first'},
'dict': {'dvoilib': 'libelle'},
'join': [{ # ERROR ! 2 dclssf pour 1 lot_id
'bdd': 'in', 'table': 'voie', 'on': ['ccodep', 'ccocom', 'ccovoi'], 'type': 'concat',
'select_cols' : ['ccodep', 'ccocom', 'codvoi', 'libvoi'],
'dict': {'libvoi': 'libelle', 'codvoi': 'ccovoi'},
}]
},{
'name': 'parcelles{}'.format(dpt_nom_tab),
'geom': {
'table_geom_in': 'geo_parcelle',
'index_geom': 'geo_parcelle'
},
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'ccovoi', 'dparpi', 'dcntpa', 'ccocomm', 'ccoprem', 'ccosecm', 'dnuplam', 'type_filiation'],
'columns_add': {
'par_id': ['ccodep', 'ccocom', 'ccopre','ccosec', 'dnupla'],
'codcom': ['ccodep', 'ccocom'],
'vl_id': ['ccodep', 'ccocom', 'ccovoi'],
'typprop_id': None },
'unique': False,
'dict': {'type_filiation': 'type'},
'join': False
},{
'name': 'lots{}'.format(dpt_nom_tab), # !!!!!! Ne trouve pas de parcelles sans lots (ex: 38357000AE0526)
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dcntpa'],
'columns_add': {
'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla'],
'par_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla'],
'dnulot': None, },
'unique': False,
'dict': {'dcntpa': 'dcntlo'},
'join': [{'bdd': 'out', 'table': 'parcelles{}'.format(dpt_nom_tab), 'on': ['par_id'], 'type': 'isin',
'select_cols' :['par_id'] }]
# },{
# 'name': 'cptprop{}'.format(dpt_nom_tab), # !!!!!! Ne trouve pas de parcelles sans lots (ex: 38357000AE0526)
# 'geom': None,
# 'drop_escape': True,
# 'columns_in' : ['ccodep', 'ccocom', 'dnupro'],
# 'columns_add': {
# 'dnupro': ['ccodep', 'ccocom', 'dnupro'],
# },
# 'unique': {'cols': ['dnupro'], 'keep': 'first'},
# 'dict': None,
# 'join': [{'bdd': 'out', 'table': 'cptprop{}'.format(dpt_nom_tab), 'on': ['dnupro'], 'type': 'notin',
# 'select_cols' :['dnupro'] }]
},]
# },{
# 'table_in' : 'suf',
# 'index_tab': 'suf',
# 'columns_in' : ['ccodep', 'ccocom', 'dnupro'],
# 'table_out': [{
# 'name': 'cptprop{}'.format(dpt_nom_tab), # !!!!!! Ne trouve pas de parcelles sans lots (ex: 38357000AE0526)
# 'geom': None,
# 'drop_escape': True,
# 'columns_in' : ['ccodep', 'ccocom', 'dnupro'],
# 'columns_add': {
# 'dnupro': ['ccodep', 'ccocom', 'dnupro'],
# },
# 'unique': {'cols': ['dnupro'], 'keep': 'first'},
# 'dict': None,
# 'join': [{'bdd': 'out', 'table': 'cptprop{}'.format(dpt_nom_tab), 'on': ['dnupro'], 'type': 'notin',
# 'select_cols' :['dnupro'] }]
# },]
},{
'table_in' : 'lots',
'index_tab': 'lots',
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnupdl', 'dcntlo', 'dnuprol'],
'table_out': [{
# 'name': 'cptprop{}'.format(dpt_nom_tab), # !!!!!! Ne trouve pas de parcelles sans lots (ex: 38357000AE0526)
# 'geom': None,
# 'drop_escape': True,
# 'columns_in' : ['ccodep', 'ccocom', 'dnuprol'],
# 'columns_add': {
# 'dnupro': ['ccodep', 'ccocom', 'dnuprol'],
# },
# 'unique': {'cols': ['dnupro'], 'keep': 'first'},
# 'dict': None,
# 'join': [{'bdd': 'out', 'table': 'cptprop{}'.format(dpt_nom_tab), 'on': ['dnupro'], 'type': 'notin',
# 'select_cols' :['dnupro'] }]
# },{
'name': 'lots{}'.format(dpt_nom_tab), # !!!!!! parcelles avec lots: existe par_id NOT IN parcelles_73
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnupdl', 'dcntlo'],
'columns_add': {
'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'],
'par_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla'],},
'unique': {'cols': ['lot_id'], 'keep': 'first'},
'dict': None,
'join': [{'bdd': 'out', 'table': 'parcelles{}'.format(dpt_nom_tab), 'on': ['par_id'], 'type': 'isin',
'select_cols' :['par_id'] }]
},{
'name': 'lots_natcult{}'.format(dpt_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'],
'columns_add': {
'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'],},
'unique': {'cols': ['lot_id'], 'keep': 'first'},
'dict': None,
'join': [{ # ERROR ! 2 dclssf pour 1 lot_id
'bdd': 'in', 'table': 'suf', 'on': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'], 'type': 'merge',
'select_cols' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot','dsgrpf','cnatsp','dclssf','ccosub','dcntsf'],
},{
'bdd': 'out', 'table': 'lots{}'.format(dpt_nom_tab), 'on': ['lot_id'], 'type': 'isin',
'select_cols' :['lot_id'] }]
},{
'name': 'cadastre{}'.format(dpt_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnuprol'],
'columns_add': {
'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'],
'dnupro': ['ccodep', 'ccocom', 'dnuprol'],},
'unique': {'cols': ['lot_id', 'dnupro'], 'keep': 'first'},
'dict': None,
'join': [{ # ERROR ! 2 dclssf pour 1 lot_id
'bdd': 'in', 'table': 'suf', 'on': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnuprol'], 'type': 'concat',
'select_cols' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnupro'], 'dict': {'dnupro': 'dnuprol'}
},{
'bdd': 'in', 'table': 'parcelle', 'on': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnuprol'], 'type': 'concat',
'select_cols' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnupro'], 'dict': {'dnupro': 'dnuprol'}
},{
'bdd': 'out', 'table': 'lots{}'.format(dpt_nom_tab), 'on': ['lot_id'], 'type': 'isin',
'select_cols' :['lot_id'] },{
'bdd': 'out', 'table': 'cptprop{}'.format(dpt_nom_tab), 'on': ['dnupro'], 'type': 'isin',
'select_cols' :['dnupro'] },]
},]
# },{
# 'table_in' : 'proprietaire', # Table source qui provient de la sortie du plugin cadastre de qgis
# 'index_tab': 'proprietaire', # Pkey de la table source
# 'columns_in': ['ccodep', 'dnuper', 'ccoqua', 'ddenom', 'jdatnss', 'dldnss', 'dsglpm', 'dlign3', 'dlign4', 'dlign5', 'dlign6', 'dnatpr', 'gtoper', 'ccogrm',
# 'ccocom', 'dnupro', 'dnomlp', 'dprnlp', 'epxnee', 'dnomcp', 'dprncp', 'ccodro', 'ccodem'],
# 'table_out': []
# },{
# 'table_in' : 'parcelle',
# 'index_tab': 'parcelle',
# 'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnupro'],
# 'table_out': [{
# 'name': 'cadastre{}'.format(dpt_nom_tab),
# 'geom': None,
# 'drop_escape': True,
# 'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnupro'],
# 'columns_add': {
# 'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla'],
# 'dnupro': ['ccodep', 'ccocom', 'dnupro'],},
# 'unique': {'cols': ['lot_id', 'dnupro'], 'keep': 'first'},
# 'dict': None,
# 'join': [{
# 'bdd': 'out', 'table': 'lots{}'.format(dpt_nom_tab), 'on': ['lot_id'], 'type': 'isin',
# 'select_cols' :['lot_id'], 'where': {'dnulot': None} },{
# 'bdd': 'out', 'table': 'cptprop{}'.format(dpt_nom_tab), 'on': ['dnupro'], 'type': 'isin',
# 'select_cols' :['dnupro'] },]
# },]
# },{
# 'table_in' : 'parcelle',
# 'index_tab': 'parcelle',
# 'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnupro'],
# 'table_out': [{
# 'name': 'cadastre{}'.format(dpt_nom_tab),
# 'geom': None,
# 'drop_escape': True,
# 'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnupro'],
# 'columns_add': {
# 'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla'],
# 'dnupro': ['ccodep', 'ccocom', 'dnupro'],},
# 'unique': {'cols': ['lot_id', 'dnupro'], 'keep': 'first'},
# 'dict': None,
# 'join': [{
# 'bdd': 'out', 'table': 'lots{}'.format(dpt_nom_tab), 'on': ['lot_id'], 'type': 'isin',
# 'select_cols' :['lot_id'], },{
# 'bdd': 'out', 'table': 'cptprop{}'.format(dpt_nom_tab), 'on': ['dnupro'], 'type': 'isin',
# 'select_cols' :['dnupro'] },]
# },]
}]
# # Connexion bdd
# bd_cad = bdd.CEN(
# user = user_cad,
# pwd = pwd_cad,
# adr = adr_cad,
# base = base_cad
# # schema = schema
# )
################################
########## Fonctions ##########
################################
start_time = dt.datetime.today()
def time_exec (init_time):
time = dt.datetime.today() - init_time
return str(time)
def replace_escape_by_0 (df):
# Remplacement des espaces dans les chaines de caractères par des 0
# if 'ccopre' in df.columns:
# df['ccopre'].replace([None, '', ' '], '000', inplace=True)
cols = ['ccopre', 'ccosec', 'dnupla', 'dparpi', 'dnuplam', 'dclssf', 'ccovoi']
for col in cols:
if col in df.columns:
df[col].replace([' '], '0', regex=True, inplace=True)
return df
def join_data (df, join, schema_in):
# Jointure des données avec une autre table
table = join['table']
bdd = join['bdd']
typ = join['type']
on = join['on']
if bdd == 'out':
con = engine_fon
sch = schema_fon
if bdd == 'in':
con = engine_cad
sch = schema_in
select_col = []
if 'select_cols' in join.keys():
select_col.extend(join['select_cols'])
if 'where' in join.keys():
select_col.extend(join['where'].keys())
tmp = pd.read_sql_table(
table_name = table,
con = con,
schema = sch,
columns = select_col
)
tmp = replace_escape_by_0(tmp)
if 'dict' in join.keys():
tmp.rename(columns=join['dict'], inplace=True)
if 'where' in join.keys():
where = join['where']
for key in where.keys():
tmp = tmp[tmp[key] == where[key] ]
if typ in ['isin', 'notin']:
# on = on[0]
for d in [df, tmp]:
d['on'] = ''
for col in on:
d['on'] += d[col].astype(str)
if typ == 'isin':
df = df[df['on'].isin(tmp['on'])]
if typ == 'notin':
df = df[~df['on'].isin(tmp['on'])]
df.drop(columns='on', inplace=True)
# if typ == 'notin':
# on = on[0]
# df = df[~df[on].isin(tmp[on])]
# df = pd.concat([df,tmp]).drop_duplicates(on, keep=False)
if typ == 'merge':
df = df.merge(tmp, on = on, how='left')
if typ == 'concat':
df = pd.concat([df,tmp], ignore_index=True).drop_duplicates()
return df
def get_geom_parcelle (df,get_geo,schema):
print('INIT import geodata ........... %s sec'%( time_exec(start_time) ))
# Définition des variables géometriques
ind_geo = get_geo['index_geom']
tab_geo = get_geo['table_geom_in']
sql = """select distinct on (t2.{0})
t2.{0},
t1.geom,
t1.supf::integer as dcntpa -- récupération de la contenance cadastrale associée car présence de géometrie non référencées dans la table "parcelles"
FROM "{1}".{2} t1
INNER JOIN (select distinct on ({0}) {0}, max(creat_date) creat_date, max(update_dat) update_dat FROM "{1}".{2} GROUP BY ({0})) t2
USING ({0}, creat_date, update_dat)""".format(ind_geo, schema, tab_geo)
tmp = gpd.read_postgis(
sql = sql,
con = engine_cad,
geom_col = 'geom',
crs = crs,
chunksize = chunk,
)
if chunk:
gdf = gpd.GeoDataFrame(pd.concat(tmp, ignore_index=True))
else:
gdf = tmp.copy()
# del tmp; gc.collect()
# gdf = tmp.copy()
del tmp
gdf.set_index(ind_geo, inplace=True)
gdf.index.name = ind_in
print('END import geodata ........... %s sec'%( time_exec(start_time) ))
print('INIT merge data - geodata ........... %s sec'%( time_exec(start_time) ))
if not gdf[gdf.dcntpa.isna()].empty:
gdf.dcntpa.fillna(0, inplace=True)
gdf['dcntpa'] = gdf['dcntpa'].astype(df.dtypes['dcntpa'].type)
# gdf = gdf.merge(df, on = [ind_in, 'dcntpa'], how='left')
tmp = gdf.merge(df, on = [ind_in, 'dcntpa'], how='right')
tmp = tmp.set_geometry('geom', drop=True, crs=crs)
tmp.rename(columns={'geometry': 'geom'}, inplace=True)
if tmp[tmp.geom.isna()].empty:
lst_ind_df = tmp[tmp.geom.isna()].index.tolist()
lst_ind_gdf = gdf.loc[gdf.index.isin(lst_ind_df)].index.tolist()
tmp.loc[tmp.index.isin(lst_ind_gdf), 'geom'] = gdf.loc[gdf.index.isin(lst_ind_gdf), 'geom']
del [gdf, df]
gdf = tmp.copy()
del tmp
export_data(gdf)
def export_data( df):
print('INIT export data TO {0}, {1} ........... {2} sec'.format(tab_out, df.shape[0], time_exec(start_time) ))
rang = [e for e in range(0, df.shape[0], chunk*5)]
for i, j in enumerate(rang):
if j == max(rang) :
jj = df.shape[0]
else:
jj = rang[i+1]
df_imp = df[j:jj].copy()
print('INIT export data TO {0} ..... {1}/{2} ...... {3} sec'.format(tab_out, jj, df.shape[0], time_exec(start_time) ))
if 'geom' in df.columns and not df[~df['geom'].isna()].empty :
df_imp = df_imp.set_geometry('geom', drop=True, crs=crs)
df_imp.rename(columns={'geometry': 'geom'}, inplace=True)
df_imp.to_postgis(
name = tab_out,
con = engine_fon,
schema = schema_fon,
index = False,
if_exists = 'append',
geom_col = 'geom',
chunksize = chunk,
)
else:
df_imp.to_sql(
name = tab_out,
con = engine_fon,
schema = schema_fon,
index = False,
if_exists = 'append',
chunksize = chunk,
method = 'multi',
)
print('END export data TO {0} ........... {1} sec'.format(tab_out, time_exec(start_time) ))
def optimize_data_frame(df):
columns = df.columns
for col in columns:
dtype = df[col].dtypes
# if dtype == 'int64' or dtype == 'int32':
len_col = len(df[col].unique())
if len_col <= df.shape[0]*0.8:
df[col] = df[col].astype('category')
return df
# Initiation des connexions bdd
engine_cad = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user_cad,pwd_cad,adr_cad,port_cad,base_cad), echo=False)
engine_fon = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user_fon,pwd_fon,adr_fon,port_fon,base_fon), echo=False)
con_cad = engine_cad.connect()
con_fon = engine_fon.connect()
################################
########## Main ##########
################################
if __name__ == "__main__":
################
# CORRECTION DUPLICATES TABLE_IN
if check_duplicates:
for DOUBLON in FIND_DOUBLON:
tab = DOUBLON['tab_in']
on_col = DOUBLON['on_col']
for col in on_col:
for dep in list_dep:
schema_in = dep + '_' + schema_cad
sql = '''
-- il existe des doublons en raison d'orthographes voisines :
-- recherche de ces doublons
SELECT DISTINCT '{0}' as insee_dep, dnuper, string_agg(DISTINCT {1},' / ') as orthographes_voisines
FROM "{2}".{3} GROUP BY dnuper HAVING count(DISTINCT {1}) > 1'''.format(dep, col, schema_in, tab)
df = pd.read_sql(
sql = sql,
con = engine_cad,
)
if df.empty:
print('No duplicate value dep {0} table {1} column {2} ====> next request'.format(dep, tab, col))
continue
for i, row in df.iterrows():
dnuper = row.dnuper
choix = row.orthographes_voisines.split(' / ')
choix = [i.strip() for i in choix]
Question = input("""Des orthographes voisines existent pour l'identifiant : {0}
dans la colonne : {1}.
Les valeurs voisines sont : {2}
Ecrire la mise à jour du champs {1} à enregistrer (c cancel) :""".format(dnuper,col, choix))
if Question.lower() == 'c' or Question.lower() == 'cancel':
continue
update = '''UPDATE "{0}".{1}
SET {2} = '{3}'
WHERE {2} like '{4}%'
AND dnuper = '{5}';'''.format(schema_in, tab, col, Question, "%' OR {} like '".format(col).join(map(str,choix)), dnuper)
try:
con_cad.execute(text(update))
print('''
Update OK !''')
except Exception as exept:
print('ERROR : {0}'.format(update))
print(exept)
sys.exit()
################
# TRUNCATE TABLE OUT
for i, DICT in enumerate(DICT_TAB):
# continue
# Définition des variables
# i = 1
# if i != 2:
# continue
tab_in = DICT_TAB[i]['table_in']
col_in = DICT_TAB[i]['columns_in']
ind_in = DICT_TAB[i]['index_tab']
tabs_out = DICT_TAB[i]['table_out']
for tab_out in reversed(tabs_out):
# continue
sql = "TRUNCATE TABLE {0}.{1} CASCADE".format(schema_fon, tab_out['name'])
print(sql)
con_fon.execute(sql)
for dep in list_dep:
schema_in = dep + '_' + schema_cad
print('''
INIT import data FROM {}
'''.format(schema_in))
################
# IMPORT IN TABLE OUT
for i, DICT in enumerate(DICT_TAB):
# Définition des variables
# i = 1
# if i != 1:
# continue
tab_in = DICT_TAB[i]['table_in']
col_in = DICT_TAB[i]['columns_in']
ind_in = DICT_TAB[i]['index_tab']
tabs_out = DICT_TAB[i]['table_out']
# Import data
print('''
INIT import data FROM {0}........... {1} sec'''.format(tab_in, time_exec(start_time) ))
tmp = pd.read_sql_table(
table_name = tab_in,
con = engine_cad,
schema = schema_in,
columns = col_in + [ind_in],
chunksize = chunk,
)
# Mise en forme des données
# start_time = dt.datetime.today()
if chunk:
DF = pd.concat(tmp, ignore_index=True)
else:
DF = tmp.copy()
DF.drop_duplicates(inplace=True)
del tmp
# DF = optimize_data_frame(DF)
DF.set_index(ind_in, inplace=True)
print('END import data ........... %s sec'%( time_exec(start_time) ))
for tab in tabs_out:
tab_out = tab['name']
dictio = tab['dict']
col_df = tab['columns_in']
col_ad = tab['columns_add']
get_geo = tab['geom']
drp_esc = tab['drop_escape']
unique = tab['unique']
join = tab['join']
# if tab_out == 'parcelles_73':
# break
# continue
print('INIT TABLE {0} ........... {1} sec'.format(tab_out, time_exec(start_time) ))
df = DF[DF.columns.intersection(col_df)].copy()
# df = optimize_data_frame(df)
# del DF; gc.collect()
# Remplacement des espaces dans les chaines de caractères par des 0
df = replace_escape_by_0(df)
if drp_esc:
df_obj = df.select_dtypes(['object'])
df[df_obj.columns] = df_obj.apply(lambda x: x.str.strip())
# df.replace([' '], '', regex=True, inplace=True)
if dictio :
df.rename(columns=dictio, inplace=True)
if join :
for j in join:
if j['bdd'] == 'in' :
# sys.exit()
df = join_data(df, j, schema_in)
if df.empty:
print('df EMPTY ====> next table')
# pass
continue
# Ajout des champs additionnels
if col_ad :
print('INIT addition columns ........... %s sec'%( time_exec(start_time) ))
for key in col_ad.keys():
if key in df.columns:
df[key + '_tmp'] = df[key].copy()
col_ad[key] = [x if x != key else key+'_tmp' for x in col_ad[key]]
aggreg = col_ad[key]
if aggreg :
df[key] = ''
for col in aggreg:
df[key] += df[col].fillna('')
# df[key] = df[aggreg].agg(''.join, axis=1)
# break
else:
df[key] = aggreg
print('ADD column {0} : {1} ........... {2} sec'.format(key,aggreg, time_exec(start_time) ))
# JOINTURE
if join :
for j in join:
if j['bdd'] == 'out' :
# break
# sys.exit()
df = join_data(df, j, schema_in)
if df.empty:
print('df EMPTY ====> next table')
# pass
continue
# sys.exit()
if unique:
df.drop_duplicates(unique['cols'], keep=unique['keep'], inplace=True)
# Conservation des champs utiles à l'insertion en bdd
name_col_out = engine_fon.dialect.get_columns(engine_fon, tab_out, schema=schema_fon)
name_col_out = [ sub['name'] for sub in name_col_out ]
if 'geom' in name_col_out and 'geom' not in df.columns:
name_col_out.remove('geom')
df = df[df.columns.intersection(name_col_out)]
####################
# Read geodataframe
# Dans le cas où un champs géometrique est nécessaire.
if get_geo:
get_geom_parcelle(df=df, get_geo=get_geo, schema=schema_in)
# print('INIT import geodata ........... %s sec'%( time_exec(start_time) ))
# # Définition des variables géometriques
# ind_geo = get_geo['index_geom']
# tab_geo = get_geo['table_geom_in']
# # Get geodata from Postgis
# # sql = "SELECT {0}, geom FROM (SELECT {0}, geom, max(creat_date), max(update_dat) FROM {1}.{2})".format(ind_geo,schema_in,tab_geo)
# sql = """select distinct on (t2.{0})
# t2.{0},
# t1.geom,
# t1.supf::integer as dcntpa -- récupération de la contenance cadastrale associée car présence de géometrie non référencées dans la table "parcelles"
# FROM "{1}".{2} t1
# INNER JOIN (select distinct on ({0}) {0}, max(creat_date) creat_date, max(update_dat) update_dat FROM "{1}".{2} GROUP BY ({0})) t2
# USING ({0}, creat_date, update_dat)""".format(ind_geo, schema_in, tab_geo)
# tmp = gpd.read_postgis(
# sql = sql,
# con = engine_cad,
# geom_col = 'geom',
# crs = crs,
# chunksize = chunk,
# )
# if chunk:
# gdf = gpd.GeoDataFrame(pd.concat(tmp, ignore_index=True))
# else:
# gdf = tmp.copy()
# # del tmp; gc.collect()
# # gdf = tmp.copy()
# del tmp; gc.collect()
# gdf.set_index(ind_geo, inplace=True)
# gdf.index.name = ind_in
# print('END import geodata ........... %s sec'%( time_exec(start_time) ))
# print('INIT merge data - geodata ........... %s sec'%( time_exec(start_time) ))
# if not gdf[gdf.dcntpa.isna()].empty:
# gdf.dcntpa.fillna(0, inplace=True)
# gdf['dcntpa'] = gdf['dcntpa'].astype(df.dtypes['dcntpa'].type)
# # gdf = gdf.merge(df, on = [ind_in, 'dcntpa'], how='left')
# tmp = gdf.merge(df, on = [ind_in, 'dcntpa'], how='right')
# tmp = tmp.set_geometry('geom', drop=True, crs=crs)
# tmp.rename(columns={'geometry': 'geom'}, inplace=True)
# if tmp[tmp.geom.isna()].empty:
# lst_ind_df = tmp[tmp.geom.isna()].index.tolist()
# lst_ind_gdf = gdf.loc[gdf.index.isin(lst_ind_df)].index.tolist()
# tmp.loc[tmp.index.isin(lst_ind_gdf), 'geom'] = gdf.loc[gdf.index.isin(lst_ind_gdf), 'geom']
# del [gdf, df]; gc.collect()
# gdf = tmp.copy()
# del tmp; gc.collect()
# export_data(gdf)
# del gdf; gc.collect()
# récupération de la liste des géometries où l'id est non présentent dans la table parcelles
# lst = gdf[gdf.par_id.isna()].index.tolist()
# # Recomposition des infos principales
# par_id = [l.replace('0','',1) for l in lst]
# gdf.loc[gdf.index.isin(lst), 'par_id'] = par_id
# gdf.loc[gdf.index.isin(lst), 'codcom'] = [l[:5] for l in par_id]
# gdf.loc[gdf.index.isin(lst), 'ccopre'] = [l[5:8] for l in par_id]
# gdf.loc[gdf.index.isin(lst), 'ccosec'] = [l[8:10] for l in par_id]
# gdf.loc[gdf.index.isin(lst), 'dnupla'] = [l[10:14] for l in par_id]
# # gdf.loc[gdf.index.isin(lst), 'vl_id'] = [l[:8] for l in par_id]
# # gdf = gdf[gdf.vl_id.str.len() == 10]
else:
export_data(df)
del df
del DF #; gc.collect()
print('END transfert data FROM département {0} ........... {1} sec'.format(dep, time_exec(start_time) ))
print('END SCRIPT')
sys.exit()
print('NOT EXIT')
0_FONCIER/foncier_insert_cadastre_V2.py
Executable
+613
View File
@@ -0,0 +1,613 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Nom : : foncier_insert_table.py
#Description : Insertion des données cadastrales à la base <foncier>.
#Copyright : 2021, CEN38
#Auteur : Colas Geier
#Version : 2.0
from sqlalchemy import create_engine, text
from geoalchemy2 import Geometry # need for pandas
import datetime as dt
import pandas as pd
import sys
# Parametrage geopandas
import geopandas as gpd
import warnings; warnings.filterwarnings('ignore', 'GeoSeries.isna', UserWarning)
check_duplicates = True
debug = True
# Parametres bdd CADASTRE (in)
# Données de sortie du plugin qgis "Cadastre"
# Les schéma ont une nomenclarture numérique du type "dep_nom"
user_cad = 'postgres' # utilisateur de connexion à la bdd
pwd_cad = 'foncier_test1' # mot de passe de la bdd
adr_cad = '172.17.0.2' # adresse ip de la bdd
port_cad = '5432' # port de la bdd
base_cad = 'postgres' # nom de la bdd
schema_cad = '202007' # nom unique qui compose le nom des schémas cibles de l'année à récupérer
list_dep = ['07', '26', '42', '38'] # liste des départements
# Parametres bdd FONCIER (out)
user_fon = 'colas_geier'
pwd_fon = 'adm1n*fOncier'
adr_fon = '91.134.194.221'
# user_fon = 'postgres' # utilisateur de connexion à la bdd
# pwd_fon = 'tutu' # mot de passe de la bdd
# adr_fon = '192.168.60.9' # adresse ip de la bdd
port_fon = '5432' # port de la bdd
base_fon = 'bd_cen' # nom de la bdd
schema_fon = 'cadastre' # nom du schéma cible dans la bdd
sufx_nom_tab = '_38' # sufixe du nom des tables où insérer la données dans le schéma cadastre (ex: _dep)
# Correspondance entre les tables
epsg = '2154' # code EPSG de la projection géographique en entrée.
crs = 'EPSG:%s'%epsg
chunk = 100000
# Il existe parfois des doublons en raison d'orthographes voisinent (entre autres)
# dans les tables de sortie du du plugin qgis "Cadastre"
FIND_DOUBLON = [{
'tab_in': 'proprietaire', # nom de la table où des doublons se trouvent
'on_col': ['ddenom', 'dprnlp', 'dldnss','jdatnss','ccogrm','dsglpm','dnatpr'] } # nom des champs où des corrections sont nécessaires
]
DICT_TAB = [{
'table_in' : 'proprietaire', # Table source qui provient de la sortie du plugin cadastre de qgis
'index_tab': 'proprietaire', # Pkey de la table source
'columns_in': ['ccodep', 'ccocom', 'dnupro', # Liste exaustive des champs à récupérer dans la table source
'dnuper', 'ccoqua', 'ddenom', 'jdatnss', 'dldnss', 'dsglpm', 'dlign3', 'dlign4', 'dlign5', 'dlign6', 'dnatpr', 'gtoper', 'ccogrm'],
'table_out': [{
'name': 'cptprop{}'.format(sufx_nom_tab), # Nom de la table cible
'geom': None, # Existance d'une géometrie à récupérer pour insertion (Non: None, Oui: cf tab_out parcelle_xx ci-dessous)
'drop_escape': False, # Supprime les champs vides à l'intérieure des chaines de carractères
'columns_in': ['ccodep', 'ccocom', 'dnupro'], # Liste des columns à récupérer en entrée.
'columns_add': {'dnupro': ['ccodep', 'ccocom', 'dnupro']}, # Définition des champs composés devant être ajoutés
'unique': {'cols': ['dnupro'], 'keep': 'first'}, # Champs devant être uniques à l'intérieur de la table en sortie, keep: ['first', 'last', False]
'dict': None, # Dictionnaire pour renommer les champs {'ancien_nom1': 'nouveau_nom1', 'ancien_nom2': 'nouveau_nom2', ...}
'join': [{ # Jointure de tables à réaliser.
'bdd': 'in', # ['in', 'out']: jointure des tables à l'import / avant export
'table': 'suf', # Nom de la table à joindre
'select_cols' : ['ccodep', 'ccocom', 'dnupro'], # Nom des champs à récupérer
'on': ['ccodep', 'ccocom', 'dnupro'], # Nom des champs sur lesquelles faire la jointure
'type': 'concat' # Type de jointure à réaliser ['concat', 'merge', 'isin', 'notin']
# 'concat': concatène les 2 tables; 'merge': merge les 2 tables en fonction du paramètre 'on'
# 'isin': Conservation des données présentent dans les 2 tables en fonction du paramètre 'on'
# 'notin': Conservation des données NON présentent dans la table 'join' en fonction du paramètre 'on'
},{
'bdd': 'in', 'table': 'lots', 'on': ['ccodep', 'ccocom', 'dnupro'], 'type': 'concat',
'select_cols' : ['ccodep', 'ccocom', 'dnuprol'],'dict': {'dnuprol': 'dnupro'}},{
'bdd': 'in', 'table': 'parcelle', 'on': ['ccodep', 'ccocom', 'dnupro'], 'type': 'concat',
'select_cols' : ['ccodep', 'ccocom', 'dnupro']},]
},{
'name': 'proprios{}'.format(sufx_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in': ['ccodep', 'dnuper', 'ccoqua', 'ddenom', 'jdatnss', 'dldnss', 'dsglpm', 'dlign3', 'dlign4', 'dlign5', 'dlign6', 'dnatpr', 'gtoper', 'ccogrm'],
'columns_add': {'dnuper': ['ccodep', 'dnuper']},
'unique': {'cols': ['dnuper'], 'keep': 'first'},
'dict': None,
'join': False
},{
'name': 'r_prop_cptprop{}'.format(sufx_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in': ['ccodep', 'dnuper', 'ccocom', 'dnupro', 'dnomlp', 'dprnlp', 'epxnee', 'dnomcp', 'dprncp', 'ccodro', 'ccodem'],
'columns_add': {
'dnuper': ['ccodep', 'dnuper'],
'dnupro': ['ccodep', 'ccocom', 'dnupro']},
'unique': {'cols': ['dnupro', 'dnuper'], 'keep': 'first'},
'dict': None,
'join': False
},]
},{
'table_in' : 'parcelle',
'index_tab': 'parcelle',
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'ccovoi', 'dparpi', 'dcntpa', 'ccocomm', 'ccoprem', 'ccosecm', 'dnuplam', 'dvoilib', 'type_filiation'],
'table_out': [{
'name': 'vl{}'.format(sufx_nom_tab),
'geom': None,
'drop_escape': False,
'columns_in' : ['ccodep', 'ccocom', 'ccovoi', 'dvoilib'],
'columns_add': {
'vl_id': ['ccodep', 'ccocom', 'ccovoi'],
'geom': None},
'unique': {'cols': ['vl_id'], 'keep': 'first'},
'dict': {'dvoilib': 'libelle'},
'join': [{
'bdd': 'in', 'table': 'voie', 'on': ['ccodep', 'ccocom', 'ccovoi'], 'type': 'concat',
'select_cols' : ['ccodep', 'ccocom', 'codvoi', 'libvoi'],
'dict': {'libvoi': 'libelle', 'codvoi': 'ccovoi'},
}]
},{
'name': 'parcelles{}'.format(sufx_nom_tab),
'geom': {
'table_geom_in': 'geo_parcelle',
'index_geom': 'geo_parcelle'
},
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'ccovoi', 'dparpi', 'dcntpa', 'ccocomm', 'ccoprem', 'ccosecm', 'dnuplam', 'type_filiation'],
'columns_add': {
'par_id': ['ccodep', 'ccocom', 'ccopre','ccosec', 'dnupla'],
'codcom': ['ccodep', 'ccocom'],
'vl_id': ['ccodep', 'ccocom', 'ccovoi'],
'typprop_id': None },
'unique': False,
'dict': {'type_filiation': 'type'},
'join': False
},{
'name': 'lots{}'.format(sufx_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dcntpa'],
'columns_add': {
'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla'],
'par_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla'],
'dnulot': None, },
'unique': False,
'dict': {'dcntpa': 'dcntlo'},
'join': [{'bdd': 'out', 'table': 'parcelles{}'.format(sufx_nom_tab), 'on': ['par_id'], 'type': 'isin',
'select_cols' :['par_id'] }]
},]
},{
'table_in' : 'lots',
'index_tab': 'lots',
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnupdl', 'dcntlo', 'dnuprol'],
'table_out': [{
'name': 'lots{}'.format(sufx_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnupdl', 'dcntlo'],
'columns_add': {
'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'],
'par_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla'],},
'unique': {'cols': ['lot_id'], 'keep': 'first'},
'dict': None,
'join': [{'bdd': 'out', 'table': 'parcelles{}'.format(sufx_nom_tab), 'on': ['par_id'], 'type': 'isin',
'select_cols' :['par_id'] }]
},{
'name': 'lots_natcult{}'.format(sufx_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'],
'columns_add': {
'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'],},
'unique': {'cols': ['lot_id'], 'keep': 'first'},
'dict': None,
'join': [{
'bdd': 'in', 'table': 'suf', 'on': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'], 'type': 'merge',
'select_cols' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot','dsgrpf','cnatsp','dclssf','ccosub','dcntsf'],
},{
'bdd': 'out', 'table': 'lots{}'.format(sufx_nom_tab), 'on': ['lot_id'], 'type': 'isin',
'select_cols' :['lot_id'] }]
},{
'name': 'cadastre{}'.format(sufx_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnuprol'],
'columns_add': {
'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'],
'dnupro': ['ccodep', 'ccocom', 'dnuprol'],},
'unique': {'cols': ['lot_id', 'dnupro'], 'keep': 'first'},
'dict': None,
'join': [{
'bdd': 'in', 'table': 'suf', 'on': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnuprol'], 'type': 'concat',
'select_cols' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnupro'], 'dict': {'dnupro': 'dnuprol'}
},{
'bdd': 'in', 'table': 'parcelle', 'on': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnuprol'], 'type': 'concat',
'select_cols' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnupro'], 'dict': {'dnupro': 'dnuprol'}
},{
'bdd': 'out', 'table': 'lots{}'.format(sufx_nom_tab), 'on': ['lot_id'], 'type': 'isin',
'select_cols' :['lot_id'] },{
'bdd': 'out', 'table': 'cptprop{}'.format(sufx_nom_tab), 'on': ['dnupro'], 'type': 'isin',
'select_cols' :['dnupro'] },]
},]
}]
################################
########## Fonctions ##########
################################
start_time = dt.datetime.today()
def time_exec (init_time):
time = dt.datetime.today() - init_time
return str(time)
def replace_escape_by_0 (df):
# Remplacement des espaces dans les chaines de caractères par des 0
# if 'ccopre' in df.columns:
# df['ccopre'].replace([None, '', ' '], '000', inplace=True)
cols = ['ccopre', 'ccosec', 'dnupla', 'dparpi', 'dnuplam', 'dclssf', 'ccovoi']
for col in cols:
if col in df.columns:
df[col].replace([' '], '0', regex=True, inplace=True)
return df
def join_data (df, join, schema_in):
# Jointure des données avec une autre table
table = join['table']
bdd = join['bdd']
typ = join['type']
on = join['on']
if bdd == 'out':
con = engine_fon
sch = schema_fon
if bdd == 'in':
con = engine_cad
sch = schema_in
select_col = []
if 'select_cols' in join.keys():
select_col.extend(join['select_cols'])
if 'where' in join.keys():
select_col.extend(join['where'].keys())
tmp = pd.read_sql_table(
table_name = table,
con = con,
schema = sch,
columns = select_col
)
tmp = replace_escape_by_0(tmp)
if 'dict' in join.keys():
tmp.rename(columns=join['dict'], inplace=True)
if 'where' in join.keys():
where = join['where']
for key in where.keys():
tmp = tmp[tmp[key] == where[key] ]
if typ in ['isin', 'notin']:
for d in [df, tmp]:
d['on'] = ''
for col in on:
d['on'] += d[col].astype(str)
if typ == 'isin':
df = df[df['on'].isin(tmp['on'])]
if typ == 'notin':
df = df[~df['on'].isin(tmp['on'])]
df.drop(columns='on', inplace=True)
if typ == 'merge':
df = df.merge(tmp, on = on, how='left')
if typ == 'concat':
df = pd.concat([df,tmp], ignore_index=True).drop_duplicates()
return df
def get_geom_parcelle (df,get_geo,schema):
print('INIT import geodata ........... %s sec'%( time_exec(start_time) ))
# Définition des variables géometriques
ind_geo = get_geo['index_geom']
tab_geo = get_geo['table_geom_in']
sql = """select distinct on (t2.{0})
t2.{0},
t1.geom,
t1.supf::integer as dcntpa -- récupération de la contenance cadastrale associée car présence de géometrie non référencées dans la table "parcelles"
FROM "{1}".{2} t1
INNER JOIN (select distinct on ({0}) {0}, max(creat_date) creat_date, max(update_dat) update_dat FROM "{1}".{2} GROUP BY ({0})) t2
USING ({0}, creat_date, update_dat)""".format(ind_geo, schema, tab_geo)
tmp = gpd.read_postgis(
sql = sql,
con = engine_cad,
geom_col = 'geom',
crs = crs,
chunksize = chunk,
)
if chunk:
gdf = gpd.GeoDataFrame(pd.concat(tmp, ignore_index=True))
else:
gdf = tmp.copy()
del tmp
gdf.set_index(ind_geo, inplace=True)
gdf.index.name = ind_in
print('END import geodata ........... %s sec'%( time_exec(start_time) ))
print('INIT merge data - geodata ........... %s sec'%( time_exec(start_time) ))
if not gdf[gdf.dcntpa.isna()].empty:
gdf.dcntpa.fillna(0, inplace=True)
gdf['dcntpa'] = gdf['dcntpa'].astype(df.dtypes['dcntpa'].type)
tmp = gdf.merge(df, on = [ind_in, 'dcntpa'], how='right')
tmp = tmp.set_geometry('geom', drop=True, crs=crs)
tmp.rename(columns={'geometry': 'geom'}, inplace=True)
if tmp[tmp.geom.isna()].empty:
lst_ind_df = tmp[tmp.geom.isna()].index.tolist()
lst_ind_gdf = gdf.loc[gdf.index.isin(lst_ind_df)].index.tolist()
tmp.loc[tmp.index.isin(lst_ind_gdf), 'geom'] = gdf.loc[gdf.index.isin(lst_ind_gdf), 'geom']
del [gdf, df]
gdf = tmp.copy()
del tmp
export_data(gdf)
def export_data(df):
print('INIT export data TO {0}, {1} ........... {2} sec'.format(tab_out, df.shape[0], time_exec(start_time) ))
rang = [e for e in range(0, df.shape[0], chunk*5)]
cd1 = 'jdatnss' in df.columns
if cd1 :
tmp = df.jdatnss.str.split(r'/',expand=True)
if tmp.shape[1] == 3 and tmp[2].str.len().max() == 4 and tmp[1].astype(int).max() <= 12:
df.jdatnss = ['%s-%s-%s'%(i[2],i[1],i[0]) for i in df.jdatnss.str.split(r'/').sort_index()]
for i, j in enumerate(rang):
if j == max(rang) :
jj = df.shape[0]
else:
jj = rang[i+1]
df_imp = df[j:jj].copy()
print('INIT export data TO {0} ..... {1}/{2} ...... {3} sec'.format(tab_out, jj, df.shape[0], time_exec(start_time) ))
if 'geom' in df.columns and not df[~df['geom'].isna()].empty :
df_imp = df_imp.set_geometry('geom', drop=True, crs=crs)
df_imp.rename(columns={'geometry': 'geom'}, inplace=True)
df_imp.to_postgis(
name = tab_out,
con = engine_fon,
schema = schema_fon,
index = False,
if_exists = 'append',
geom_col = 'geom',
chunksize = chunk,
)
else:
df_imp.to_sql(
name = tab_out,
con = engine_fon,
schema = schema_fon,
index = False,
if_exists = 'append',
chunksize = chunk,
method = 'multi',
)
print('END export data TO {0} ........... {1} sec'.format(tab_out, time_exec(start_time) ))
def optimize_data_frame(df):
columns = df.columns
for col in columns:
# dtype = df[col].dtypes
# if dtype == 'int64' or dtype == 'int32':
len_col = len(df[col].unique())
if len_col <= df.shape[0]*0.8:
df[col] = df[col].astype('category')
return df
# Initiation des connexions bdd
engine_cad = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user_cad,pwd_cad,adr_cad,port_cad,base_cad), echo=False)
engine_fon = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user_fon,pwd_fon,adr_fon,port_fon,base_fon), echo=False)
con_cad = engine_cad.connect()
con_fon = engine_fon.connect()
################################
########## Main ##########
################################
if __name__ == "__main__":
################
# CORRECTION DUPLICATES TABLE_IN
################
if check_duplicates:
for DOUBLON in FIND_DOUBLON:
tab = DOUBLON['tab_in']
on_col = DOUBLON['on_col']
for col in on_col:
for dep in list_dep:
schema_in = dep + '_' + schema_cad
sql = '''
-- il existe des doublons en raison d'orthographes voisines :
-- recherche de ces doublons
SELECT DISTINCT '{0}' as insee_dep, dnuper, string_agg(DISTINCT {1},' / ') as orthographes_voisines
FROM "{2}".{3} GROUP BY dnuper HAVING count(DISTINCT {1}) > 1'''.format(dep, col, schema_in, tab)
df = pd.read_sql(
sql = sql,
con = engine_cad,
)
if df.empty:
print('No duplicate value dep {0} table {1} column {2} ====> next request'.format(dep, tab, col))
continue
for i, row in df.iterrows():
dnuper = row.dnuper
choix = row.orthographes_voisines.split(' / ')
choix = [i.strip() for i in choix]
Question = input("""Des orthographes voisines existent pour l'identifiant : {0}
dans la colonne : {1}.
Les valeurs voisines sont : {2}
Ecrire la mise à jour du champs {1} à enregistrer (c cancel) :""".format(dnuper,col, choix))
if Question.lower() == 'c' or Question.lower() == 'cancel':
continue
update = '''UPDATE "{0}".{1}
SET {2} = '{3}'
WHERE {2} like '{4}%'
AND dnuper = '{5}';'''.format(schema_in, tab, col, Question, "%' OR {} like '".format(col).join(map(str,choix)), dnuper)
try:
con_cad.execute(text(update))
print('''
Update OK !''')
except Exception as exept:
print('ERROR : {0}'.format(update))
print(exept)
sys.exit()
################
# TRUNCATE TABLE OUT
################
if not debug:
for i, DICT in enumerate(DICT_TAB):
tab_in = DICT_TAB[i]['table_in']
col_in = DICT_TAB[i]['columns_in']
ind_in = DICT_TAB[i]['index_tab']
tabs_out = DICT_TAB[i]['table_out']
for tab_out in reversed(tabs_out):
sql = "TRUNCATE TABLE {0}.{1} CASCADE".format(schema_fon, tab_out['name'])
print(sql)
con_fon.execute(sql)
################
# IMPORT IN TABLE OUT
################
for dep in list_dep:
schema_in = dep + '_' + schema_cad
print('''
INIT import data FROM {}
'''.format(schema_in))
for i, DICT in enumerate(DICT_TAB):
tab_in = DICT_TAB[i]['table_in']
col_in = DICT_TAB[i]['columns_in']
ind_in = DICT_TAB[i]['index_tab']
tabs_out = DICT_TAB[i]['table_out']
# Import data
print('''
INIT import data FROM {0}........... {1} sec'''.format(tab_in, time_exec(start_time) ))
tmp = pd.read_sql_table(
table_name = tab_in,
con = engine_cad,
schema = schema_in,
columns = col_in + [ind_in],
chunksize = chunk,
)
# Mise en forme des données
if chunk:
DF = pd.concat(tmp, ignore_index=True)
else:
DF = tmp.copy()
DF.drop_duplicates(inplace=True)
del tmp
DF.set_index(ind_in, inplace=True)
print('END import data ........... %s sec'%( time_exec(start_time) ))
for tab in tabs_out:
tab_out = tab['name']
dictio = tab['dict']
col_df = tab['columns_in']
col_ad = tab['columns_add']
get_geo = tab['geom']
drp_esc = tab['drop_escape']
unique = tab['unique']
join = tab['join']
print('INIT TABLE {0} ........... {1} sec'.format(tab_out, time_exec(start_time) ))
df = DF[DF.columns.intersection(col_df)].copy()
# Remplacement des espaces dans les chaines de caractères par des 0
df = replace_escape_by_0(df)
if drp_esc:
df_obj = df.select_dtypes(['object'])
df[df_obj.columns] = df_obj.apply(lambda x: x.str.strip())
if dictio :
df.rename(columns=dictio, inplace=True)
if join :
for j in join:
if j['bdd'] == 'in' :
df = join_data(df, j, schema_in)
if df.empty:
print('df EMPTY ====> next table')
continue
# Ajout des champs additionnels
if col_ad :
print('INIT addition columns ........... %s sec'%( time_exec(start_time) ))
for key in col_ad.keys():
if key in df.columns:
df[key + '_tmp'] = df[key].copy()
col_ad[key] = [x if x != key else key+'_tmp' for x in col_ad[key]]
aggreg = col_ad[key]
if aggreg :
df[key] = ''
for col in aggreg:
df[key] += df[col].fillna('')
else:
df[key] = aggreg
print('ADD column {0} : {1} ........... {2} sec'.format(key,aggreg, time_exec(start_time) ))
if tab_out == 'proprios%s'%sufx_nom_tab:
add = pd.DataFrame(data={'dnuper': ['%sY99999'%dep], 'ddenom': ['PROPRIETAIRES NON RENSEIGNES']})
df = pd.concat([df,add])
# JOINTURE
if join :
for j in join:
if j['bdd'] == 'out' :
df = join_data(df, j, schema_in)
if df.empty:
print('df EMPTY ====> next table')
continue
if unique:
df.drop_duplicates(unique['cols'], keep=unique['keep'], inplace=True)
# Conservation des champs utiles à l'insertion en bdd
name_col_out = engine_fon.dialect.get_columns(engine_fon, tab_out, schema=schema_fon)
name_col_out = [ sub['name'] for sub in name_col_out ]
if 'geom' in name_col_out and 'geom' not in df.columns:
name_col_out.remove('geom')
df = df[df.columns.intersection(name_col_out)]
####################
# Read geodataframe
# Dans le cas où un champs géometrique est nécessaire.
if get_geo:
get_geom_parcelle(df=df, get_geo=get_geo, schema=schema_in)
else:
export_data(df)
# del df
# Si insertion de données dans r_prop_cptprop,
# attribution d'un compte 'PROPRIETAIRES NON RENSEIGNES' au cptprop sans proprio
if tab_out == 'r_prop_cptprop%s'%sufx_nom_tab:
print('''ATTRIBUTION d'un compte 'PROPRIETAIRES NON RENSEIGNES' au cptprop sans proprio''')
df = pd.read_sql_table(
con = engine_fon,
table_name = 'cptprop%s'%sufx_nom_tab,
schema = schema_fon,
columns = ['dnupro'] )
tmp = pd.read_sql_table(
con = engine_fon,
table_name = 'r_prop_cptprop%s'%sufx_nom_tab,
schema = schema_fon,
columns = ['dnupro'] )
df = df[~df.dnupro.isin(tmp.dnupro)]
if df.empty:
print('TOUS les propriétaires sont renseignés ===> NEXT !')
continue
df['dnuper'] = '%sY99999'%dep
export_data(df)
del df
del DF
print('END transfert data FROM département {0} ........... {1} sec'.format(dep, time_exec(start_time) ))
print('END SCRIPT')
0_FONCIER/foncier_insert_cadastre_V3.py
+631
View File
@@ -0,0 +1,631 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from pycen import con_fon
from sqlalchemy.sql import text
from sqlalchemy.engine import URL
from sqlalchemy import create_engine
import pandas as pd
import geopandas as gpd
# user = 'cgeier'
# pwd = 'adm1n*bdCen'
# adr = '91.134.194.221'
# base = 'cadastre'
user = 'postgres'
pwd = 'foncier_test1'
adr = '172.17.0.2'
base = 'postgres'
url = URL.create('postgresql+psycopg2',
username=user,
password=pwd,
host=adr,
database=base,
)
con_cad = create_engine(url)
# from pycen import con_cad
# sql = '''SELECT * FROM "38_202207".suf WHERE parcelle = '3800740000B0705';'''
# df = pd.read_sql_query(sql,con_cad)
# df.drop_duplicates(inplace=True)
# df.annee = '2020'
# df.to_sql('suf',con_cad,"38_202207",if_exists='append',index=False)
# fadd = '/home/colas/Documents/tmp/FONCIER_FEDE/add_parc.gpkg'
# add = gpd.read_file(fadd)
def recup_cols_table(table,con,schema='38_202207'):
lst_cols = con.dialect.get_columns(con,table,schema)
return [x['name'] for x in lst_cols]
def __get_pkey__(engine,table_name,schema):
pk = engine.dialect.get_pk_constraint(engine,table_name=table_name,schema=schema)
return pk
def _where_parcelle(sql0,schema,list_parid):
if list_parid is not None:
chunk = None
if isinstance(list_parid,str):
list_parid = [list_parid]
LIST_ID = str(tuple(list_parid)).replace(',)',')')
sql1 = 'AND' if 'WHERE' in sql0 else 'WHERE'
sql1 += '''
p.parcelle IN {list_id}
;'''.format(sch=schema,list_id=LIST_ID)
else :
chunk = 200000
sql1 = ';'
sql = sql0 + sql1
df = pd.read_sql_query(text(sql),con=con_cad,chunksize=chunk)
# if chunk is not None:
# for d in df:
# print(d.shape[0])
# d.drop_duplicates(inplace=True)
# print(d.drop_duplicates().shape[0])
# else :
# df.drop_duplicates(inplace=True)
return df
def _get_chunk(df1,df2):
# cptp1 = pd.DataFrame()
list_DF1 = []
for d1 in df1:
list_DF1.append(d1)
DF1 = pd.concat(list_DF1)
# cptp2 = pd.DataFrame()
list_DF2 = []
for d2 in df2:
list_DF2.append(d2)
DF2 = pd.concat(list_DF2)
return pd.concat([DF1,DF2]).drop_duplicates()
def __get_parcelles__(sql0,list_parid):
if list_parid is not None:
chunk = None
if isinstance(list_parid,str):
list_parid = [list_parid]
LIST_ID = str(tuple(list_parid)).replace(',)',')')
sql1 = '''
WHERE t1.geo_parcelle IN {list_id}
;'''.format(list_id=LIST_ID)
else :
chunk = None
sql1 = ';'
sql = sql0 + sql1
# print(text(sql))
return gpd.read_postgis(sql=sql,con=con_cad,chunksize=chunk)
def _get_parcelles1(schema='38_202207',list_parid=None):
sql0 = '''SELECT DISTINCT ON (t1.geo_parcelle)
t1.geo_parcelle,
case when t1.geom is null then t2.geom
else t1.geom
end geom,
substring(p.parcelle from 1 for 2)||substring(p.parcelle from 4 for 12) par_id,
substring(p.parcelle from 1 for 2)||substring(p.parcelle from 4 for 3) codcom,
substring(p.parcelle from 1 for 2) ccodep,
substring(p.parcelle from 4 for 3) ccocom,
substring(p.parcelle from 7 for 3) ccopre,
substring(p.parcelle from 10 for 2) ccosec,
substring(p.parcelle from 12 for 4) dnupla,
p.annee annee_pci,
t1.update_dat,
p.dparpi,
p.dcntpa,
p.ccocomm,
p.ccoprem,
p.ccosecm,
p.dnuplam,
p.ccovoi,
p.ccoriv,
p.type_filiation "type",
substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 3)||p.ccovoi vl_id,
(SELECT STRING_AGG(DISTINCT gtoper::text,',') FROM "{sch}".proprietaire WHERE (ccodep,ccocom,dnupro) = (t3.ccodep,t3.ccocom,t3.dnupro)) gtoper,
(SELECT STRING_AGG(DISTINCT ccogrm::text,',') FROM "{sch}".proprietaire WHERE (ccodep,ccocom,dnupro) = (t3.ccodep,t3.ccocom,t3.dnupro)) ccogrm,
(SELECT STRING_AGG(DISTINCT CONCAT(gtoper::text||COALESCE('_'||ccogrm::text,'')),',') FROM "{sch}".proprietaire WHERE (ccodep,ccocom,dnupro) = (t3.ccodep,t3.ccocom,t3.dnupro)) ccogrm,
(SELECT STRING_AGG(DISTINCT TRIM(ddenom)::text,',') FROM "{sch}".proprietaire WHERE (ccodep,ccocom,dnupro) = (t3.ccodep,t3.ccocom,t3.dnupro)) ddenom
FROM "{sch}".{t1} p
LEFT JOIN "{sch}".parcelle_info t2 ON t2.geo_parcelle = p.parcelle
LEFT JOIN "{sch}"."geo_parcelle" t1 ON t1.geo_parcelle = p.parcelle
LEFT JOIN "{sch}".proprietaire t3 USING (ccodep,ccocom,dnupro)
'''.format(
sch=schema,
t1='parcelle')
return __get_parcelles__(sql0,list_parid)
def _get_parcelles2(schema='38_202207',list_parid=None):
sql0 = '''SELECT DISTINCT ON (t1.geo_parcelle)
t1.geo_parcelle,
substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) par_id,
substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 3) codcom,
substring(t1.geo_parcelle from 1 for 2) ccodep,
substring(t1.geo_parcelle from 4 for 3) ccocom,
substring(t1.geo_parcelle from 7 for 3) ccopre,
substring(t1.geo_parcelle from 10 for 2) ccosec,
substring(t1.geo_parcelle from 12 for 4) dnupla,
t1.annee annee_pci,
t1.update_dat,
p.dparpi,
p.dcntpa,
p.ccocomm,
p.ccoprem,
p.ccosecm,
p.dnuplam,
p.ccovoi,
p.ccoriv,
p.type_filiation "type",
t1.geom,
substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 3)||p.ccovoi vl_id,
(SELECT STRING_AGG(DISTINCT gtoper::text,',') FROM "{sch}".proprietaire WHERE comptecommunal = t3.comptecommunal) gtoper,
(SELECT STRING_AGG(DISTINCT ccogrm::text,',') FROM "{sch}".proprietaire WHERE comptecommunal = t3.comptecommunal) ccogrm,
(SELECT STRING_AGG(DISTINCT CONCAT(gtoper::text||COALESCE('_'||ccogrm::text,'')),',') FROM "{sch}".proprietaire WHERE comptecommunal = t3.comptecommunal) ccogrm,
(SELECT STRING_AGG(DISTINCT TRIM(ddenom)::text,',') FROM "{sch}".proprietaire WHERE comptecommunal = t3.comptecommunal) ddenom
FROM "{sch}"."{t1}" t1
LEFT JOIN ("{sch}".parcelle_info t2
LEFT JOIN "{sch}".proprietaire t3 USING (comptecommunal))
USING (geo_parcelle)
LEFT JOIN "{sch}".parcelle p ON t1.geo_parcelle = p.parcelle
'''.format(
sch=schema,
t1='geo_parcelle')
return __get_parcelles__(sql0,list_parid)
def _get_parcelles(schema='38_202207',list_parid=None):
p1 = _get_parcelles1(schema,list_parid)
print('parcelles from parcelle .......... OK')
p2 = _get_parcelles2(schema,list_parid)
print('parcelles from geo_parcelle ...... OK')
return _get_chunk(p1,p2)
def _get_voie2(schema='38_202207',list_parid=None):
sql0 = '''
SELECT
t1.ccodep||t1.ccocom||p.ccovoi vl_id,
t1.libvoi libelle
FROM "{sch}"."{t1}" t1
JOIN "{sch}".parcelle p USING (voie)
'''.format(
sch=schema,
t1='voie')
return _where_parcelle(sql0,schema,list_parid)
def _get_lots_natcult1(schema='38_202207',list_parid=None):
sql0 = '''
SELECT
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||btrim(dnulot) as lot_id, -- Identifiant du lot (c'est bon, quand il n'y a pas de lot, c'est la même façon de noter l'identifiant)
btrim(dsgrpf) as dsgrpf , -- Sous-groupe de nature de culture
btrim(cnatsp) as cnatsp , -- Code nature de culture spéciale
CASE WHEN trim(dclssf) = '' OR dclssf = '00'
THEN NULL::integer
ELSE ltrim(dclssf,'0')::integer
END AS dclssf, -- Classe dans le groupe et la série tarif
btrim(ccosub) as ccosub , -- Lettres indicatives de la suf
dcntsf -- Contenance de la suf
FROM "{sch}"."{t1}"
'''.format(
sch=schema,
t1='suf')
return _where_parcelle(sql0,schema,list_parid)
def _get_lots_natcult2(schema='38_202207',list_parid=None):
sql0 = '''
SELECT
CASE WHEN TRIM(dnulot) = ''
THEN substring(parcelle from 1 for 2)||substring(parcelle from 4 for 12)||'0000000'
ELSE substring(parcelle from 1 for 2)||substring(parcelle from 4 for 12)||TRIM(dnulot)
END lot_id,
parcelle,
substring(parcelle from 1 for 2)||substring(parcelle from 4 for 12) par_id,
dsgrpf,
cnatsp,
dclssf,
ccosub,
dcntsf
FROM "{sch}"."{t1}"
JOIN "{sch}".parcelle p USING(parcelle)
'''.format(
sch=schema,
t1='suf')
return _where_parcelle(sql0,schema,list_parid)
def _get_lots01(schema='38_202207'):
sql0 = '''
SELECT DISTINCT
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot as lot_id, -- Identifiant du lot character varying(21)
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla as par_id, -- Identifiant de la parcelle
dnulot, -- Numéro du lot
max(dcntlo) as dcntlo -- Contenance cadastrale (m²) on prend contenance maxi quand même lot_id mais dcntlo différentes
FROM "{sch}".{t1}
JOIN "{sch}".geo_parcelle ON lots.parcelle = geo_parcelle.geo_parcelle -- on ne garde que les lots sur des parcelles dont on a la géométrie
WHERE
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot IN
( -- on sélectionne uniquement les lots qui supportent une subdivision fiscale pour exclure les lots batis
SELECT ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot FROM "{sch}".suf WHERE btrim(dnulot) <> '')
GROUP BY ccodep, ccocom, ccopre, ccosec, dnupla, dnulot
'''.format(
sch=schema,
t1='lots')
return _where_parcelle(sql0,schema,None)
def _get_lots02(schema='38_202207'):
sql0 = '''
SELECT
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot as lot_id, -- Identifiant du lot character varying(21)
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla as par_id, -- Identifiant de la parcelle
dnulot, -- Numéro du lot
sum(dcntsf) as dcntlo -- Contenance cadastrale (m²) = On additionne les contenances des différentes suf qui composent le lot
FROM "{sch}".suf
JOIN "{sch}".geo_parcelle ON suf.parcelle = geo_parcelle.geo_parcelle -- on ne garde que les sufs sur des parcelles dont on a la géométrie
WHERE
btrim(dnulot) <> '' AND ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot NOT IN (
SELECT DISTINCT ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot FROM "{sch}".lots WHERE btrim(dnulot)<>'')
GROUP BY ccodep, ccocom, ccopre, ccosec, dnupla, dnulot
'''.format(
sch=schema,
t1='lots')
return _where_parcelle(sql0,schema,None)
def _get_lots03(schema='38_202207'):
sql0 = '''
-- on sélectionne les parcelles qui supportent une subdivision fiscale (unique ou multiple) même si divisées en lots car dans ce cas ce sont surement des copropriétés (lots batis/jardins...)
SELECT DISTINCT
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla as lot_id, -- Identifiant du lot fictif(=id parcelle)
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla as par_id, -- Identifiant de la parcelle
null as dnulot, -- Numéro du lot
dcntpa as dcntlo-- Contenance cadastrale (m²)
FROM "{sch}".parcelle
JOIN "{sch}".geo_parcelle ON parcelle.parcelle = geo_parcelle.geo_parcelle -- on ne garde que les les parcelles dont on a la géométrie
WHERE
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla IN
(SELECT ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||trim(dnulot) FROM "{sch}".suf); -- toutes les parcelles dont dnulot est NULL
'''.format(
sch=schema,
t1='lots')
return _where_parcelle(sql0,schema,None)
def _get_lots1(schema='38_202207'):
l1 = _get_lots01(schema)
l2 = _get_lots02(schema)
l3 = _get_lots03(schema)
return pd.concat([*l1,*l2,*l3])
def _get_lots2(schema='38_202207',list_parid=None):
sql0 = '''set work_mem='265MB';
SELECT DISTINCT
CASE WHEN TRIM(t.dnulot) = '' OR TRIM(t.dnulot) IS NULL
THEN substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 12)||TRIM(t.ccosub)
ELSE substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 12)||TRIM(t.dnulot)
END lot_id,
t.parcelle,
substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 12) par_id,
CASE WHEN TRIM(t.dnulot) = '' OR TRIM(t.dnulot) IS NULL
THEN TRIM(t.ccosub)
ELSE TRIM(t.dnulot)
END dnulot,
CASE WHEN t.dcntsf IS NULL
THEN l.dcntlo
ELSE t.dcntsf
END dcntlo,
geo_sub1.creat_date,
CASE WHEN geo_sub1.geom IS NULL
THEN geo_sub2.geom
ELSE geo_sub1.geom
END geom
FROM "{sch}"."{t1}" t
JOIN "{sch}".parcelle p USING (parcelle)
LEFT JOIN "{sch}".geo_parcelle geo_p ON geo_p.geo_parcelle = p.parcelle
LEFT JOIN "{sch}".lots l USING (parcelle)
--LEFT JOIN "{sch}".lotslocaux ll USING (lots)
LEFT JOIN "{sch}".geo_subdfisc_parcelle geo_sub_p ON p.parcelle = geo_sub_p.geo_parcelle
--LEFT JOIN "{sch}".geo_subdfisc geo_sub USING (geo_subdfisc)
LEFT JOIN "{sch}".geo_subdfisc geo_sub1
ON (geo_sub1.geo_subdfisc = geo_sub_p.geo_subdfisc AND LOWER(geo_sub1.tex) = LOWER(t.ccosub))
LEFT JOIN "{sch}".geo_subdfisc geo_sub2
ON (geo_sub2.geo_subdfisc = geo_sub_p.geo_subdfisc AND TRIM(LOWER(geo_sub2.tex)) = '')
WHERE ST_INTERSECTS(geo_sub1.geom,geo_p.geom) OR ST_INTERSECTS(geo_sub2.geom,geo_p.geom)
'''.format(
sch=schema,
t1='suf')
return _where_parcelle(sql0,schema,list_parid)
def _get_cadastre2(schema='38_202207',list_parid=None):
sql0 = '''
SELECT DISTINCT
CASE WHEN TRIM(t.dnulot) = ''
THEN substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 12)||'0000000'
ELSE substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 12)||TRIM(t.dnulot)
END lot_id,
substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 3)||t.dnupro dnupro
FROM "{sch}"."{t1}" t
JOIN "{sch}".parcelle p
LEFT JOIN "{sch}".lots l USING (parcelle)
USING (parcelle)
'''.format(
sch=schema,
t1='suf')
return _where_parcelle(sql0,schema,list_parid)
def _get_cptprop1(schema='38_202207',list_parid=None):
sql0 = '''
WITH lot as (
SELECT ccodep,ccocom,dnuprol dnupro
FROM "{sch}".lots
)
SELECT DISTINCT
t.ccodep||t.ccocom||t.dnupro dnupro,
t."annee" annee_matrice
FROM "{sch}"."{t1}" t
--JOIN "{sch}".suf s USING (ccodep,ccocom,dnupro)
--JOIN lot l USING (ccodep,ccocom,dnupro)
JOIN "{sch}".parcelle p USING (ccodep,ccocom,dnupro)
'''.format(
sch=schema,
t1='proprietaire')
return _where_parcelle(sql0,schema,list_parid)
def _get_cptprop2(schema='38_202207',list_parid=None):
sql0 = '''
SELECT DISTINCT
substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 3)||t.dnupro dnupro,
t.annee annee_matrice
FROM "{sch}"."{t1}" t
JOIN "{sch}".parcelle p
LEFT JOIN "{sch}".lots l USING (parcelle)
USING (parcelle)
'''.format(
sch=schema,
t1='suf')
return _where_parcelle(sql0,schema,list_parid)
def _get_cptprop(schema='38_202207',list_parid=None):
cptprop1 = _get_cptprop1(schema=schema, list_parid=list_parid)
print('cptprop from proprietaire ... OK')
cptprop2 = _get_cptprop2(schema=schema, list_parid=list_parid)
print('cptprop from suf ............ OK')
return _get_chunk(cptprop1,cptprop2)
def _get_r_prop_cptprop1(schema='38_202207',list_parid=None):
sql0 = '''
SELECT DISTINCT
substring(p.parcelle from 1 for 2)||substring(p.parcelle from 4 for 3)||t.dnupro dnupro,
substring(p.parcelle from 1 for 2)||t.dnuper dnuper,
ccodro,
ccodem
FROM "{sch}"."{t1}" t
JOIN "{sch}".parcelle p USING (ccodep,ccocom, dnupro)
'''.format(
sch=schema,
t1='proprietaire')
return _where_parcelle(sql0,schema,list_parid)
def _get_proprios1(schema='38_202207',list_parid=None):
sql0 = '''
SELECT DISTINCT
t.ccodep||t.dnuper dnuper,
TRIM(ccoqua)::int ccoqua,
TRIM(ddenom) ddenom,
TRIM(dqualp) dqualp,
TRIM(dnomlp) dnomlp,
TRIM(dnomus) dnomus,
TRIM(dprnus) dprnus,
TRIM(dprnlp) dprnlp,
TRIM(epxnee) epxnee,
TRIM(dnomcp) dnomcp,
TRIM(dprncp) dprncp,
TRIM(jdatnss) jdatnss,
TRIM(dldnss) dldnss,
TRIM(dlign3) dlign3,
TRIM(dlign4) dlign4,
TRIM(dlign5) dlign5,
TRIM(dlign6) dlign6,
TRIM(gtoper)::int gtoper,
TRIM(ccogrm)::int ccogrm,
TRIM(dnatpr) dnatpr,
TRIM(dsglpm) dsglpm,
t."annee" annee_matrice
FROM "{sch}"."{t1}" t
JOIN "{sch}".parcelle p USING (ccodep,ccocom, dnupro)
'''.format(
sch=schema,
t1='proprietaire')
return _where_parcelle(sql0,schema,list_parid)
def _to_cadaste_table(df,con,pkey,table,schema):
# Si la clé-primaire de la table est > 1 colonne
if len(pkey) > 1 :
lstid = str(
tuple(
df[pkey]\
.drop_duplicates()\
.itertuples(index=False, name=None)
)
).replace(',)',')')
pk = str(tuple(pkey)).replace("'",'"')
# Si la clé-primaire de la table est 1 colonne
else:
pk = pkey[0]
lstid = str(tuple(df[pk].drop_duplicates()))\
.replace(',)',')')
# Récupération des données déjà en BDD
sql = '''
SELECT * FROM {sch}.{tab}
WHERE {id} IN {lst}
;'''.format(
sch = schema,
tab = table,
id = pk,
lst = lstid
)
indb = pd.read_sql_query(sql,con,index_col=pkey)
# Exclusion des données déjà en base
df.set_index(pkey,inplace=True)
todb = df[
~df.index.isin(indb.index)
].copy()
df.reset_index(drop=False, inplace=True)
todb.reset_index(drop=False, inplace=True)
indb.reset_index(drop=False, inplace=True)
# Envoie du tableau dans la bdd si non vide
if not todb.empty:
# Adaptation des types des champs si nécessaire
dtyp = {}
if 'geom' in todb.columns:
from geoalchemy2 import Geometry
todb = todb.to_wkt()
dtyp = {'geom':Geometry(geometry_type='MULTIPOLYGON',srid=2154)}
cols = todb.columns[todb.columns.isin(indb.columns)]
todb[cols]\
.to_sql(
table,
con,
schema,
if_exists='append',
index=False,
dtype = dtyp
)
print('INSERT %s news data ! OK'%todb.shape[0])
else :
print('NO news data to insert !')
def to_vl(df,con):
table = 'vl'
schema = 'cadastre'
pkey = __get_pkey__(con,table,schema)['constrained_columns']
_to_cadaste_table(df,con,pkey,table,schema)
def to_parcelles(df,con):
table = 'parcelles'
schema = 'cadastre'
pkey = __get_pkey__(con,table,schema)['constrained_columns']
_to_cadaste_table(df,con,pkey,table,schema)
def to_lots(df,con):
table = 'lots'
schema = 'cadastre'
pkey = __get_pkey__(con,table,schema)['constrained_columns']
_to_cadaste_table(df,con,pkey,table,schema)
def to_lots_natcult(df,con):
table = 'lots_natcult'
schema = 'cadastre'
# pkey = __get_pkey__(con,table,schema)['constrained_columns']
pkey = ['lot_id']
_to_cadaste_table(df,con,pkey,table,schema)
def to_cadastre(df,con):
table = 'cadastre'
schema = 'cadastre'
# pkey = __get_pkey__(con,table,schema)['constrained_columns']
pkey = ['lot_id','dnupro']
_to_cadaste_table(df,con,pkey,table,schema)
def to_r_prop_cptprop(df,con):
table = 'r_prop_cptprop'
schema = 'cadastre'
pkey = __get_pkey__(con,table,schema)['constrained_columns']
_to_cadaste_table(df,con,pkey,table,schema)
def to_proprios(df,con):
table = 'proprios'
schema = 'cadastre'
pkey = __get_pkey__(con,table,schema)['constrained_columns']
_to_cadaste_table(df,con,pkey,table,schema)
def to_cptprop(df,con):
table = 'cptprop'
schema = 'cadastre'
pkey = __get_pkey__(con,table,schema)['constrained_columns']
_to_cadaste_table(df,con,pkey,table,schema)
if __name__ == "__main__":
from pycen import update_to_sql
# par = '3805050000E0523'
par = None
sch = '38_202007'
# cptprop1 = _get_cptprop1(schema=sch, list_parid=par)
# cptprop2 = _get_cptprop2(schema=sch, list_parid=par)
cptprop = _get_cptprop(schema=sch, list_parid=par)
proprios = _get_proprios1(schema=sch, list_parid=par)
r_prop_cptprop = _get_r_prop_cptprop1(schema=sch, list_parid=par)
voie = _get_voie2(schema=sch, list_parid=par)
# parcelles = _get_parcelles2(schema=sch, list_parid=par)
parcelles = _get_parcelles(schema=sch, list_parid=par)
lots = _get_lots2(schema=sch, list_parid=par)
lots_natcult = _get_lots_natcult2(schema=sch, list_parid=par)
cadastre = _get_cadastre2(schema=sch, list_parid=par)
to_cptprop(cptprop,con_fon)
to_proprios(proprios,con_fon)
to_r_prop_cptprop(r_prop_cptprop,con_fon)
to_vl(voie,con_fon)
to_parcelles(parcelles,con_fon)
to_lots(lots,con_fon)
to_lots_natcult(lots_natcult,con_fon)
to_cadastre(cadastre,con_fon)
0_FONCIER/foncier_insert_cadastre_V4.py
+855
View File
@@ -0,0 +1,855 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from pycen import con_fon
from sqlalchemy.sql import text
from sqlalchemy.engine import URL
from sqlalchemy import create_engine
import pandas as pd
import geopandas as gpd
user = 'postgres'
pwd = 'foncier_test1'
adr = '172.17.0.2'
base = 'postgres'
url = URL.create('postgresql+psycopg2',
username=user,
password=pwd,
host=adr,
database=base,
)
con_cad = create_engine(url)
def recup_cols_table(table,con,schema='38_202207'):
lst_cols = con.dialect.get_columns(con,table,schema)
return [x['name'] for x in lst_cols]
def __get_pkey__(engine,table_name,schema):
pk = engine.dialect.get_pk_constraint(engine,table_name=table_name,schema=schema)
return pk
def _where_parcelle(sql0,schema,list_parid):
if list_parid is not None:
chunk = None
if isinstance(list_parid,str):
list_parid = [list_parid]
LIST_ID = str(tuple(list_parid)).replace(',)',')')
sql1 = '''
WHERE p.parcelle IN {list_id}
;'''.format(sch=schema,list_id=LIST_ID)
else :
chunk = 200000
sql1 = ';'
sql = sql0 + sql1
df = pd.read_sql_query(text(sql),con=con_cad,chunksize=chunk)
# if chunk is not None:
# for d in df:
# print(d.shape[0])
# d.drop_duplicates(inplace=True)
# print(d.drop_duplicates().shape[0])
# else :
# df.drop_duplicates(inplace=True)
return df
def _get_chunk(df1,df2):
# cptp1 = pd.DataFrame()
list_DF1 = []
for d1 in df1:
list_DF1.append(d1)
DF1 = pd.concat(list_DF1)
# cptp2 = pd.DataFrame()
list_DF2 = []
for d2 in df2:
list_DF2.append(d2)
DF2 = pd.concat(list_DF2)
return pd.concat([DF1,DF2]).drop_duplicates()
def __get_parcelles__(sql0,list_parid):
if list_parid is not None:
chunk = None
if isinstance(list_parid,str):
list_parid = [list_parid]
LIST_ID = str(tuple(list_parid)).replace(',)',')')
sql1 = '''
WHERE t1.geo_parcelle IN {list_id}
;'''.format(list_id=LIST_ID)
else :
chunk = None
sql1 = ';'
sql = sql0 + sql1
# print(text(sql))
return gpd.read_postgis(sql=sql,con=con_cad,chunksize=chunk)
def _get_parcelles1(schema='38_202207',list_parid=None):
sql0 = '''SELECT DISTINCT ON (t1.geo_parcelle)
t1.geo_parcelle,
case when t1.geom is null then t2.geom
else t1.geom
end geom,
substring(p.parcelle from 1 for 2)||substring(p.parcelle from 4 for 12) par_id,
substring(p.parcelle from 1 for 2)||substring(p.parcelle from 4 for 3) codcom,
substring(p.parcelle from 1 for 2) ccodep,
substring(p.parcelle from 4 for 3) ccocom,
substring(p.parcelle from 7 for 3) ccopre,
substring(p.parcelle from 10 for 2) ccosec,
substring(p.parcelle from 12 for 4) dnupla,
p.annee annee_pci,
t1.update_dat,
p.dparpi,
p.dcntpa,
p.ccocomm,
p.ccoprem,
p.ccosecm,
p.dnuplam,
p.ccovoi,
p.ccoriv,
p.type_filiation "type",
substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 3)||p.ccovoi vl_id,
(SELECT STRING_AGG(DISTINCT gtoper::text,',') FROM "{sch}".proprietaire WHERE (ccodep,ccocom,dnupro) = (t3.ccodep,t3.ccocom,t3.dnupro)) gtoper,
(SELECT STRING_AGG(DISTINCT ccogrm::text,',') FROM "{sch}".proprietaire WHERE (ccodep,ccocom,dnupro) = (t3.ccodep,t3.ccocom,t3.dnupro)) ccogrm,
(SELECT STRING_AGG(DISTINCT CONCAT(gtoper::text||COALESCE('_'||ccogrm::text,'')),',') FROM "{sch}".proprietaire WHERE (ccodep,ccocom,dnupro) = (t3.ccodep,t3.ccocom,t3.dnupro)) ccogrm,
(SELECT STRING_AGG(DISTINCT TRIM(ddenom)::text,',') FROM "{sch}".proprietaire WHERE (ccodep,ccocom,dnupro) = (t3.ccodep,t3.ccocom,t3.dnupro)) ddenom
FROM "{sch}".{t1} p
LEFT JOIN "{sch}".parcelle_info t2 ON t2.geo_parcelle = p.parcelle
LEFT JOIN "{sch}"."geo_parcelle" t1 ON t1.geo_parcelle = p.parcelle
LEFT JOIN "{sch}".proprietaire t3 USING (ccodep,ccocom,dnupro)
'''.format(
sch=schema,
t1='parcelle')
return __get_parcelles__(sql0,list_parid)
def drop_tables(con):
sql = '''
TRUNCATE TABLE cadastre.parcelles RESTART IDENTITY CASCADE;
TRUNCATE TABLE cadastre.cadastre RESTART IDENTITY CASCADE;
TRUNCATE TABLE cadastre.cptprop RESTART IDENTITY CASCADE;
TRUNCATE TABLE cadastre.r_prop_cptprop RESTART IDENTITY CASCADE;
TRUNCATE TABLE cadastre.lots RESTART IDENTITY CASCADE;
TRUNCATE TABLE cadastre.lots_natcult RESTART IDENTITY CASCADE;
TRUNCATE TABLE cadastre.proprios RESTART IDENTITY CASCADE;
TRUNCATE TABLE cadastre.vl RESTART IDENTITY CASCADE;
ALTER SEQUENCE
cadastre.cadastre_cad_id_seq RESTART WITH 1;
ALTER SEQUENCE
cadastre.lots_natcult_lotnatcult_id_seq RESTART WITH 1;
'''
with con.begin() as cnx:
cnx.execute(sql)
def where_parid(parid):
if not parid: return ""
return "AND parcelle.parcelle {} {}".format(
"IN" if isinstance(parid, (list,pd.Series)) and len(parid) > 1 else "=",
tuple(parid) if isinstance(parid, (list,pd.Series)) and len(parid) > 1
else "'%s'"%parid[0] if isinstance(parid, (list,pd.Series)) and len(parid) == 1
else "'%s'"%parid
)
# CENRA 2020
def _insert_voie1(schema='38_202207',list_parid=None):
'''Insertion des voies et lieux-dits dans cadastre.vl à partir de voie.
On ne garde que les voies qui sont effectivement affectées à des parcelles pour ne pas se trainer les voies annulées (annul = 'O')
'''
sql0 = '''set work_mem='512MB';
INSERT into cadastre.vl
SELECT
v.ccodep||v.ccocom||v.codvoi AS vl_id, --character varying(10) NOT NULL, -->>> c'est pas le bon compte de caractères, mais je n'ai pas de doublons en auvergne
v.libvoi AS libelle,--character varying(50),
null AS geom -- geom geometry(MultiPolygon,2154),
FROM "{sch}".voie v
LEFT JOIN "{sch}".parcelle ON v.ccodep||v.ccocom||v.codvoi = parcelle.ccodep||parcelle.ccocom||parcelle.ccovoi
WHERE v.ccodep||v.ccocom||v.codvoi IN (
SELECT DISTINCT ccodep||ccocom||ccovoi FROM "{sch}".parcelle )
'''.format(
sch=schema)
if list_parid:
sql0 += where_parid(list_parid)
with con_fon.begin() as cnx:
cnx.execute(sql0)
def _insert_voie2(schema='38_202207',list_parid=None):
'''Insertions des vois manquantes à partir de leur ccoriv --> 12
'''
sql0 = '''set work_mem='512MB';
INSERT into cadastre.vl
SELECT DISTINCT
parcelle.ccodep||parcelle.ccocom||parcelle.ccovoi AS vl_id,
libvoi
FROM "{sch}".parcelle
JOIN "{sch}".voie ON voie.ccoriv = parcelle.ccoriv AND voie.ccocom = parcelle.ccocom
WHERE parcelle.ccodep||parcelle.ccocom||parcelle.ccovoi NOT IN (
SELECT ccodep||ccocom||codvoi AS vl_id FROM "{sch}".voie)
'''.format(
sch=schema)
if list_parid:
sql0 += where_parid(list_parid)
with con_fon.begin() as cnx:
cnx.execute(sql0)
def _insert_voie3(schema='38_202207',list_parid=None):
'''Ajout manuel des dernières voies référencées dans parcelle mais introuvables dans voie (ni par codvoi ni par ccoriv) --> 0
'''
sql0 = '''set work_mem='512MB';
INSERT into cadastre.vl
SELECT DISTINCT ccodep||ccocom||ccovoi AS vl_id, 'NON TROUVEE' AS libelle
FROM "{sch}".parcelle
WHERE ccodep||ccocom||ccovoi NOT IN (SELECT vl_id from cadastre.vl)
'''.format(
sch=schema)
if list_parid:
sql0 += where_parid(list_parid)
with con_fon.begin() as cnx:
cnx.execute(sql0)
def insert_voie(schema='38_202207',list_parid=None):
_insert_voie1(schema,list_parid)
_insert_voie2(schema,list_parid)
_insert_voie3(schema,list_parid)
print('INSERT voie OK')
# def _manage_parc_without_geom(schema='38_202207'):
# sql = '''CREATE TEMP TABLE parcelles_sans_geom AS (
# SELECT parcelle FROM "{sch}".parcelle a LEFT JOIN "{sch}".geo_parcelle b ON b.geo_parcelle = a.parcelle WHERE b.geo_parcelle is null
# '''.format(
# sch=schema)
# with con_fon.begin() as cnx:
# cnx.execute(sql)
# def add_old_geom(schema='38_202207'):
# sql = '''
# '''
def _insert_parcelle1(schema='38_202207',list_parid=None):
'''01/ parcelles'''
sql = '''set work_mem='512MB';
INSERT INTO cadastre.parcelles
(
SELECT DISTINCT ON (ccodep || ccocom || replace(ccopre, ' ', '0') || replace(ccosec, ' ', '0') || dnupla)
ccodep || ccocom || replace(ccopre, ' ', '0') || replace(ccosec, ' ', '0') || dnupla AS par_id,
-- Identifiant de la parcelle
ccodep || ccocom AS codcom,
-- Code insee de la commune
--CASE WHEN btrim(ccovoi) <> '' THEN ccodep || ccocom || ccovoi ELSE NULL END AS vl_id,
ccodep || ccocom || ccovoi AS vl_id,
--character varying(10) NOT NULL, -->>> on ne met rien lorsque que ccovoi non renseigné dans la table parcelle
btrim(ccopre) as ccopre,
-- Préfixe de section ou quartier servi
btrim(ccosec) as ccosec,
-- Code de la section
CASE WHEN btrim(dnupla) = ''
THEN NULL :: INTEGER
ELSE ltrim(btrim(dnupla), '0') :: INTEGER END AS dnupla,
-- Numéro de la parcelle
CASE WHEN btrim(dparpi) = ''
THEN NULL :: INTEGER
ELSE ltrim(btrim(dparpi), '0') :: INTEGER END AS dparpi,
-- Ancien numéro de la parcelle, parcelle primitive
dcntpa :: INTEGER,
-- Contenance cadastrale de la parcelle
NULL AS typprop_id,
-- Type de propriété de la parcelle >>>>>>>>>>>>>>>>>>>>>>>>>>>>>> à remplir après (Cf. tâches de mise à jour du cadastre)
geom,
-- Géométrie de la parcelle
ccocomm,
-- Commune Parcelle mère
ccoprem,
-- Préfixe Parcelle mère
btrim(ccosecm) as ccosecm,
-- Section Parcelle mère
CASE WHEN dnuplam = ''
THEN NULL :: INTEGER
ELSE ltrim(dnuplam, '0') :: INTEGER
END AS dnuplam,
-- Numéro Parcelle mère
type_filiation AS "type", -- Type de filiation
jdatat
FROM "{sch}".parcelle
JOIN "{sch}".geo_parcelle ON geo_parcelle.geo_parcelle = parcelle.parcelle
'''.format(
sch=schema)
if list_parid:
sql += where_parid(list_parid).replace('AND','WHERE')
sql += ');'
with con_fon.begin() as cnx:
cnx.execute(sql)
# A REPRENDRE
def _insert_parcelle2(schema='38_202207',list_parid=None):
'''2/ Géométries manquantes
Parcelles MAJIC qui n'ont plus de geom dans EDIGEO parce qu'elles ont été fusionnées, divisées ou transférées
Il y a la geom des nouvelles parcelles dans EDIGEO mais rien dans MAJIC
On récupère la geom sur un EDIGEO antérieur'''
sql = '''set work_mem='512MB';
INSERT INTO cadastre.parcelles
( WITH t1 as (
SELECT DISTINCT ON (a.ccodep || a.ccocom || replace(a.ccopre, ' ', '0') || replace(a.ccosec, ' ', '0') || a.dnupla)
parcelle,
a.ccodep || a.ccocom || replace(a.ccopre, ' ', '0') || replace(a.ccosec, ' ', '0') || a.dnupla AS par_id,
-- Identifiant de la parcelle
a.ccodep || a.ccocom AS codcom,
-- Code insee de la commune
--CASE WHEN btrim(ccovoi) <> '' THEN ccodep || ccocom || ccovoi ELSE NULL END AS vl_id,
a.ccodep || a.ccocom || a.ccovoi AS vl_id,
--character varying(10) NOT NULL, -->>> on ne met rien lorsque que ccovoi non renseigné dans la table parcelle
replace(a.ccopre, ' ', '0') AS ccopre,
-- Préfixe de section ou quartier servi
-- btrim(a.ccosec) AS ccosec,
replace(a.ccosec, ' ', '0') AS ccosec,
-- Code de la section
CASE WHEN btrim(a.dnupla) = ''
THEN NULL :: INTEGER
ELSE ltrim(btrim(a.dnupla), '0') :: INTEGER
END AS dnupla,
-- Numéro de la parcelle
CASE WHEN btrim(a.dparpi) = ''
THEN NULL :: INTEGER
ELSE ltrim(btrim(a.dparpi), '0') :: INTEGER
END AS dparpi,
-- Ancien numéro de la parcelle, parcelle primitive
a.dcntpa :: INTEGER,
-- Contenance cadastrale de la parcelle
NULL AS typprop_id,
-- Type de propriété de la parcelle >>>>>>>>>>>>>>>>>>>>>>>>>>>>>> à remplir après (Cf. tâches de mise à jour du cadastre)
-- parcelles_cen.geom,
-- Géométrie de la parcelle
a.ccocomm,
-- Commune Parcelle mère
a.ccoprem,
-- Préfixe Parcelle mère
btrim(a.ccosecm) AS ccosecm,
-- Section Parcelle mère
CASE WHEN a.dnuplam = ''
THEN NULL :: INTEGER
ELSE ltrim(a.dnuplam, '0') :: INTEGER END AS dnuplam,
-- Numéro Parcelle mère
a.type_filiation AS "type", -- Type de filiation
a.jdatat
FROM "{sch}".parcelle a )
SELECT
t1.par_id,
t1.codcom,
t1.vl_id,
t1.ccopre,
t1.ccosec,
t1.dnupla,
t1.dparpi,
t1.dcntpa,
t1.typprop_id,
CASE WHEN b.geom IS NULL THEN parcelles_cen.geom
ELSE b.geom END geom,
t1.ccocomm,
t1.ccoprem,
t1.ccosecm,
t1.dnuplam,
t1.type,
t1.jdatat
FROM t1
LEFT JOIN "{sch}".parcelle_info b ON t1.parcelle = b.geo_parcelle -- les parcelles sans geom
LEFT JOIN "{sch}".parcelle c ON t1.parcelle = c.parcelle -- les parcelles sans geom
LEFT JOIN cadastre.parcelles_cen ON t1.par_id = parcelles_cen.par_id
WHERE t1.par_id NOT IN (SELECT par_id FROM cadastre.parcelles)
)
'''.format(
sch=schema)
if list_parid:
sql += where_parid(list_parid)
with con_fon.begin() as cnx:
cnx.execute(sql)
def insert_parcelle(schema='38_202207',list_parid=None):
_insert_parcelle1(schema,list_parid)
_insert_parcelle2(schema,list_parid)
print('INSERT parcelle OK')
def checkortho_proprio_sym(con,colonne,schema='38_202207'):
sql = '''
SELECT DISTINCT '{dep}' AS insee_dep, dnuper, string_agg(DISTINCT TRIM({column}),' / ') AS orthographes_voisines
FROM "{sch}".proprietaire
GROUP BY dnuper HAVING count(DISTINCT TRIM({column})) > 1
'''.format(dep = schema[:2],
sch=schema,
column=colonne)
return pd.read_sql(sql,con=con)
def update_synonyme_proprio(con,data,colonne,schema='38_202207'):
data['split_ortho'] = (data.orthographes_voisines
.str.replace("'","''")
.str.split(' / '))
for i,row in data.iterrows():
sql = '''UPDATE "{sch}".proprietaire SET {column} = '{dd1}' WHERE btrim({column}) {sym} {dd2} AND dnuper = '{dnup}';'''.format(
sch=schema,
dnup=row.dnuper,
column=colonne,
dd1=row.split_ortho[0],
sym='IN' if len(row.split_ortho[1:]) > 1 else '=',
dd2=tuple(row.split_ortho[1:]) if len(row.split_ortho[1:]) > 1 else "'%s'"% row.split_ortho[1])
with con.begin() as cnx:
cnx.execute(sql)
def check_proprio(schema='38_202207',con=con_fon):
is_synonyme = checkortho_proprio_sym(con,'ddenom',schema)
if not is_synonyme.empty:
update_synonyme_proprio(con,is_synonyme,'ddenom',schema)
is_diff_lab = checkortho_proprio_sym(con,'dldnss',schema)
if not is_diff_lab.empty:
update_synonyme_proprio(con,is_diff_lab,'dldnss',schema)
is_incomplete = checkortho_proprio_sym(con,'jdatnss',schema)
if not is_incomplete.empty:
update_synonyme_proprio(con,is_incomplete,'jdatnss',schema)
is_diff_ccogrm = checkortho_proprio_sym(con,'ccogrm',schema)
if not is_diff_ccogrm.empty:
update_synonyme_proprio(con,is_diff_ccogrm,'ccogrm',schema)
is_diff_dsglpm = checkortho_proprio_sym(con,'dsglpm',schema)
if not is_diff_dsglpm.empty:
update_synonyme_proprio(con,is_diff_dsglpm,'dsglpm',schema)
is_diff_dnatpr = checkortho_proprio_sym(con,'dnatpr',schema)
if not is_diff_dnatpr.empty:
update_synonyme_proprio(con,is_diff_dnatpr,'dnatpr',schema)
is_diff_dprnus = checkortho_proprio_sym(con,'dprnus',schema)
if not is_diff_dprnus.empty:
update_synonyme_proprio(con,is_diff_dprnus,'dprnus',schema)
is_diff_dprnlp = checkortho_proprio_sym(con,'dprnlp',schema)
if not is_diff_dprnlp.empty:
update_synonyme_proprio(con,is_diff_dprnlp,'dprnlp',schema)
is_diff_dnomus = checkortho_proprio_sym(con,'dnomus',schema)
if not is_diff_dnomus.empty:
update_synonyme_proprio(con,is_diff_dnomus,'dnomus',schema)
def _insert_proprio(schema='38_202207',list_parid=None):
sql = '''set work_mem='512MB';
INSERT INTO cadastre.proprios
SELECT DISTINCT
ccodep||dnuper dnuper,
ccoqua::integer,
btrim(ddenom) AS ddenom,
jdatnss,
btrim(dldnss) AS dldnss,
btrim(dsglpm) AS dsglpm,
btrim(dlign3) AS dlign3,
btrim(dlign4) AS dlign4,
btrim(dlign5) AS dlign5,
btrim(dlign6) AS dlign6,
btrim(dnatpr) AS dnatpr,
gtoper::integer,
ccogrm::integer,
btrim(dqualp) AS dnatpr,
btrim(dnomlp) AS dnomlp,
btrim(dprnlp) AS dprnlp,
btrim(dnomus) AS dnomus,
btrim(dprnus) AS dprnus,
btrim(dforme) AS dforme
FROM "{sch}".proprietaire
'''.format(
sch=schema)
with con_fon.begin() as cnx:
cnx.execute(sql)
def insert_no_proprio(schema='38_202207'):
sql = '''
INSERT INTO cadastre.proprios (dnuper, ddenom)
VALUES ('{dep}Y99999', ' PROPRIETAIRES NON RENSEIGNES');
'''.format(
dep=schema[:2])
with con_fon.begin() as cnx:
cnx.execute(sql)
def insert_proprio(schema='38_202207'):
check_proprio(schema,con_fon)
_insert_proprio(schema)
insert_no_proprio(schema)
print('INSERT proprio OK')
def _insert_cptprop1(schema='38_202207'):
'''
Insertion des comptes de propriété depuis la table proprietaire
'''
sql = '''set work_mem='512MB';
INSERT INTO cadastre.cptprop
SELECT DISTINCT ccodep || ccocom || dnupro
FROM "{sch}".proprietaire WHERE btrim(dnupro) <> ''
'''.format(
sch=schema)
with con_fon.begin() as cnx:
cnx.execute(sql)
def _insert_cptprop2(schema='38_202207'):
'''
Insertion des comptes de propriété utilisés dans
lots/parcelles/suf mais non référencés dans proprietaire
'''
sql = '''set work_mem='512MB';
INSERT INTO cadastre.cptprop
SELECT DISTINCT ccodep || ccocom || dnuprol
FROM "{sch}".lots WHERE comptecommunal NOT IN (SELECT DISTINCT comptecommunal FROM "{sch}".proprietaire)
UNION
SELECT DISTINCT ccodep || ccocom || dnupro
FROM "{sch}".parcelle WHERE comptecommunal NOT IN (SELECT DISTINCT comptecommunal FROM "{sch}".proprietaire)
UNION
SELECT DISTINCT ccodep || ccocom || dnupro
FROM "{sch}".suf WHERE comptecommunal NOT IN (SELECT DISTINCT comptecommunal FROM "{sch}".proprietaire)
'''.format(
sch=schema)
with con_fon.begin() as cnx:
cnx.execute(sql)
def insert_cptprop(schema='38_202207'):
_insert_cptprop1(schema)
_insert_cptprop2(schema)
print('INSERT cptprop OK')
def _insert_r_prop_cptprop0(schema='38_202207'):
sql = '''set work_mem='512MB';
INSERT INTO cadastre.r_prop_cptprop
SELECT DISTINCT
ccodep||dnuper,
ccodep || ccocom || dnupro,
btrim(dnomlp),
btrim(dprnlp),
CASE WHEN btrim(epxnee) = ''
THEN NULL::text
ELSE btrim(epxnee)
END AS epxnee ,
btrim(dnomcp),
btrim(dprncp),
btrim(ccodro),
btrim(ccodem)
FROM "{sch}".proprietaire WHERE btrim(dnupro) <> ''
'''.format(
sch=schema)
with con_fon.begin() as cnx:
cnx.execute(sql)
def _insert_r_prop_cptprop1(schema='38_202207'):
sql = '''set work_mem='512MB';
INSERT INTO cadastre.r_prop_cptprop
SELECT DISTINCT '{dep}Y99999', ccodep || ccocom || dnuprol
FROM "{sch}".lots WHERE comptecommunal NOT IN (SELECT DISTINCT comptecommunal FROM "{sch}".proprietaire)
UNION
SELECT DISTINCT '{dep}Y99999', ccodep || ccocom || dnupro
FROM "{sch}".parcelle WHERE comptecommunal NOT IN (SELECT DISTINCT comptecommunal FROM "{sch}".proprietaire)
UNION
SELECT DISTINCT '{dep}Y99999', ccodep || ccocom || dnupro
FROM "{sch}".suf WHERE comptecommunal NOT IN (SELECT DISTINCT comptecommunal FROM "{sch}".proprietaire)
'''.format(
sch=schema,
dep=schema[:2]
)
with con_fon.begin() as cnx:
cnx.execute(sql)
def insert_r_prop_cptprop(schema='38_202207'):
_insert_r_prop_cptprop0(schema)
_insert_r_prop_cptprop1(schema)
print('INSERT r_prop_cptprop OK')
def _insert_lot1(schema='38_202207',list_parid=None):
sql = '''set work_mem='512MB';
INSERT INTO cadastre.lots (lot_id, par_id, dnulot, dcntlo)
--les parcelles divisées en lots de la table lots
(WITH parcelle AS (
SELECT DISTINCT
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot AS id_lot, -- Identifiant du lot character varying(21)
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla AS par_id, -- Identifiant de la parcelle
dnulot, -- Numéro du lot
max(dcntlo) AS dcntlo -- Contenance cadastrale (m²) on prend contenance maxi quand même id_lot mais dcntlo différentes
FROM "{sch}".lots
JOIN "{sch}".geo_parcelle ON lots.parcelle = geo_parcelle.geo_parcelle -- on ne garde que les lots sur des parcelles dont on a la géométrie
WHERE ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot IN
( -- on sélectionne uniquement les lots qui supportent une subdivision fiscale pour exclure les lots batis
SELECT ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot FROM "{sch}".suf WHERE btrim(dnulot) <> '')
GROUP BY ccodep, ccocom, ccopre, ccosec, dnupla, dnulot
UNION -- les lots de la table suf qui ne sont pas dans la table lots
SELECT DISTINCT
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot AS id_lot, -- Identifiant du lot character varying(21)
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla AS par_id, -- Identifiant de la parcelle
dnulot, -- Numéro du lot
sum(dcntsf) AS dcntlo -- Contenance cadastrale (m²) = On additionne les contenances des différentes suf qui composent le lot
FROM "{sch}".suf
JOIN "{sch}".geo_parcelle ON suf.parcelle = geo_parcelle.geo_parcelle -- on ne garde que les sufs sur des parcelles dont on a la géométrie
WHERE btrim(dnulot) <> '' AND ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot NOT IN (
SELECT DISTINCT ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot FROM "{sch}".lots WHERE btrim(dnulot)<>'')
GROUP BY ccodep, ccocom, ccopre, ccosec, dnupla, dnulot
UNION -- les parcelles à lot unique
-- on sélectionne les parcelles qui supportent une subdivision fiscale (unique ou multiple) même si divisées en lots car dans ce cas ce sont surement des copropriétés (lots batis/jardins...)
SELECT DISTINCT
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla AS id_lot, -- Identifiant du lot fictif(=id parcelle)
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla AS par_id, -- Identifiant de la parcelle
null AS dnulot, -- Numéro du lot
dcntpa AS dcntlo-- Contenance cadastrale (m²)
FROM "{sch}".parcelle
JOIN "{sch}".geo_parcelle ON parcelle.parcelle = geo_parcelle.geo_parcelle -- on ne garde que les parcelles dont on a la géométrie
WHERE ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla IN
(SELECT ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||trim(dnulot) FROM "{sch}".suf) -- toutes les parcelles dont dnulot est NULL
)
SELECT id_lot, par_id, dnulot, dcntlo
FROM parcelle
'''.format(
sch=schema)
if list_parid:
list_parid = [x[:2] + x[3:] for x in list_parid]
sql += (where_parid(list_parid)
.replace('AND', 'WHERE')
.replace('.parcelle','.par_id')
)
sql += ');'
with con_fon.begin() as cnx:
cnx.execute(sql)
def _insert_lot2(schema='38_202207'):
sql = '''set work_mem='512MB';
insert into cadastre.lots (lot_id, par_id, dnulot, dcntlo,geom)
with t1 as (
SELECT DISTINCT
-- CASE WHEN TRIM(t.dnulot) = '' OR TRIM(t.dnulot) IS NULL
-- THEN substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 12)||TRIM(t.ccosub)
-- ELSE substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 12)||TRIM(t.dnulot)
-- END lot_id,
--substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 12)||TRIM(t.dnulot) lot_id,
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||TRIM(dnupla)||TRIM(dnulot) AS lot_id,
parcelle,
t.ccosub,
t.dcntsf,
--substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 12) par_id,
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||TRIM(dnupla) AS par_id,
CASE WHEN TRIM(t.dnulot) = '' OR TRIM(t.dnulot) IS NULL
THEN TRIM(t.ccosub)
ELSE TRIM(t.dnulot)
END dnulot
FROM "{sch}"."suf" t
)
select distinct on (t1.lot_id,t1.par_id)
t1.lot_id,
t1.par_id,
t1.dnulot,
CASE WHEN t1.dcntsf IS NULL
THEN l.dcntlo
ELSE t1.dcntsf
END dcntlo,
CASE WHEN geo_sub1.geom IS NULL
THEN geo_sub2.geom
ELSE geo_sub1.geom
END geom
from t1
JOIN "{sch}".parcelle p USING (parcelle)
LEFT JOIN "{sch}".geo_parcelle geo_p ON geo_p.geo_parcelle = p.parcelle
LEFT JOIN "{sch}".lots l USING (parcelle)
--LEFT JOIN "{sch}".lotslocaux ll USING (lots)
LEFT JOIN "{sch}".geo_subdfisc_parcelle geo_sub_p ON p.parcelle = geo_sub_p.geo_parcelle
--LEFT JOIN "{sch}".geo_subdfisc geo_sub USING (geo_subdfisc)
LEFT JOIN "{sch}".geo_subdfisc geo_sub1
ON (geo_sub1.geo_subdfisc = geo_sub_p.geo_subdfisc AND LOWER(geo_sub1.tex) = LOWER(t1.ccosub))
LEFT JOIN "{sch}".geo_subdfisc geo_sub2
ON (geo_sub2.geo_subdfisc = geo_sub_p.geo_subdfisc AND TRIM(LOWER(geo_sub2.tex)) = '')
WHERE (t1.par_id,t1.lot_id) not IN ( select par_id,lot_id from cadastre.lots )
OR t1.par_id not IN ( select par_id from cadastre.lots )
order by 2,1,5
;
'''.format(
sch=schema)
with con_fon.begin() as cnx:
cnx.execute(sql)
def insert_lot(schema='38_202207'):
_insert_lot1(schema)
_insert_lot2(schema)
print('INSERT lot OK')
def insert_cadastre(schema='38_202207',list_parid=None):
sql = '''
SET work_mem='512MB';
INSERT into cadastre.cadastre (lot_id, dnupro)
( WITH t1 as (
--les pdl
SELECT DISTINCT
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot AS lot_id, -- Identifiant du lot character varying(21)
ccodep || ccocom || dnuprol AS dnupro
FROM "{sch}".lots
JOIN "{sch}".geo_parcelle ON lots.parcelle = geo_parcelle.geo_parcelle -- on ne garde que les lots sur des parcelles dont on a la géométrie
WHERE
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||trim(dnulot) IN ( -- on sélectionne uniquement les lots qui supportent une subdivision fiscale
SELECT ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||trim(dnulot)
FROM "{sch}".suf)
UNION
-- les lots de la table suf qui ne sont pas dans la table lots
SELECT DISTINCT
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot AS id_lot, -- Identifiant du lot character varying(21)
ccodep || ccocom || dnupro AS dnupro
FROM "{sch}".suf
JOIN "{sch}".geo_parcelle ON suf.parcelle = geo_parcelle.geo_parcelle -- on ne garde que les sufs sur des parcelles dont on a la géométrie
WHERE
trim(dnulot) <> '' AND ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||btrim(dnulot)
NOT IN (
SELECT DISTINCT ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||btrim(dnulot) FROM "{sch}".lots)
UNION
---les parcelles sans lot
SELECT DISTINCT
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla AS id_lot, -- Identifiant du lot fictif(=id parcelle)
ccodep || ccocom || dnupro AS dnupro
FROM "{sch}".parcelle
JOIN "{sch}".geo_parcelle ON parcelle.parcelle = geo_parcelle.geo_parcelle -- on ne garde que les parcelles dont on a la géométrie
WHERE
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla IN
(SELECT ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||btrim(dnulot) FROM "{sch}".suf) -- toutes les parcelles dont dnulot est NULL
)
SELECT lot_id, dnupro FROM t1
'''.format(
sch=schema)
if list_parid:
sql += 'JOIN cadastre.lots l USING (lot_id) '
list_parid = [x[:2] + x[3:] for x in list_parid]
sql += (where_parid(list_parid)
.replace('AND', 'WHERE')
.replace('parcelle.parcelle','l.par_id')
)
sql += ');'
with con_fon.begin() as cnx:
cnx.execute(sql)
print('INSERT cadastre OK')
def cadastre_missing(con=con_fon):
sql = '''set work_mem='512MB';
INSERT into cadastre.cadastre (lot_id, dnupro)
SELECT DISTINCT
lot_id, -- Identifiant du lot character varying(21)
dnupro
FROM cadastre.cadastre_cen
WHERE lot_id NOT IN (SELECT lot_id FROM cadastre.cadastre)
AND lot_id IN (SELECT lot_id FROM cadastre.lots)
AND dnupro IN (SELECT dnupro FROM cadastre.cptprop)
;
'''
with con.begin() as cnx:
cnx.execute(sql)
def insert_lotnatcult(schema='38_202207',list_parid=None):
sql = '''set work_mem='512MB';
INSERT INTO cadastre.lots_natcult (lot_id, dsgrpf, cnatsp, dclssf, ccosub, dcntsf)
SELECT DISTINCT
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||btrim(dnulot) AS lot_id, -- Identifiant du lot (c'est bon, quand il n'y a pas de lot, c'est la même façon de noter l'identifiant)
btrim(dsgrpf) AS dsgrpf , -- Sous-groupe de nature de culture
btrim(cnatsp) AS cnatsp , -- Code nature de culture spéciale
CASE WHEN trim(dclssf) = '' OR dclssf = '00'
THEN NULL::integer
ELSE ltrim(dclssf,'0')::integer
END AS dclssf, -- Classe dans le groupe et la série tarif
btrim(ccosub) AS ccosub , -- Lettres indicatives de la suf
dcntsf -- Contenance de la suf
FROM "{sch}".suf
WHERE
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||trim(dnulot) IN (SELECT lot_id FROM cadastre.lots)
'''.format(
sch=schema)
if list_parid:
sql += (where_parid(list_parid)
.replace('parcelle.parcelle','suf.suf')
)
with con_fon.begin() as cnx:
cnx.execute(sql)
print('INSERT lotnatcult OK')
def update_typprop(con=con_fon):
sql = '''set work_mem='512MB';
UPDATE
cadastre.parcelles
SET typprop_id = nullif(concat_ws('_',gtoper::text,ccogrm::text),'')
FROM cadastre.proprios
JOIN cadastre.r_prop_cptprop USING (dnuper)
JOIN cadastre.cptprop USING (dnupro)
JOIN cadastre.cadastre USING (dnupro)
JOIN cadastre.lots USING (lot_id)
WHERE lots.par_id = parcelles.par_id
AND typprop_id is NULL
'''
with con.begin() as cnx:
cnx.execute(sql)
print('UPDATE typprop OK')
if __name__ == "__main__":
from pycen import update_to_sql
# par = '3805050000E0523'
par = None
sch = '38_202603'
drop_tables(con_fon)
lst_sch = [
# '07_202603',
'26_202603','42_202603','38_202603']
for sch in lst_sch:
print(' INIT ',sch)
insert_voie(sch)
insert_parcelle(sch)
insert_proprio(sch)
insert_cptprop(sch)
insert_r_prop_cptprop(sch)
insert_lot(sch)
insert_cadastre(sch)
insert_lotnatcult(sch)
cadastre_missing(con_fon)
update_typprop(con_fon)
# pg_restore -h 172.17.0.2 -U postgres --dbname="bd_cen" -c /home/colas/Documents/9_PROJETS/0_FONCIER/DUMP/bd_cen_20240418_16h50_beforemaj2023.dump
_insert_lot1('38_202504','38405000ZD0209')
insert_cadastre('38_202504','380405000ZD0209')
insert_lotnatcult('38_202504','380405000ZD0209')
0_FONCIER/foncier_insert_site.py
Executable
+221
View File
@@ -0,0 +1,221 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Nom : : foncier_insert_table.py
#Description : Insertion des données cadastrales à la base <foncier> après de sa création.
#Copyright : 2021, CEN38
#Auteur : Colas Geier
#Version : 1.0
import pandas as pd
import geopandas as gpd
from sqlalchemy import create_engine
from geoalchemy2 import Geometry, shape
from shapely import wkb
from shapely.geometry.multipolygon import MultiPolygon
from shapely.ops import unary_union
import datetime as dt
import sys
import gc
# Parametres bdd cen_38 #bart (in)
# user_in = 'cen_admin'
# pwd_in = '#CEN38@venir'
# adr_in = '192.168.0.3'
# port_in = '5432'
# base_in = 'bd_cen38'
from pycen import con_bdcen
schema_in = 'sites'
table_in = 'c_sites_zonages'
# Parametres bdd FONCIER NEW (out)
# user_out = 'postgres'
# pwd_out = 'tutu'
# adr_out = '192.168.60.9'
# port_out = '5432'
# base_out = 'bd_cen'
from pycen import con_fon
schema_out = 'sites'
# Parametres généraux
crs = 'EPSG:2154'
chunk = None
# Définition des fonctions
start_time = dt.datetime.today()
def time_exec (init_time):
time = dt.datetime.today() - init_time
return str(time)
def drop_exist(df, con, table, schema, chunk=None):
exist = pd.read_sql_table(
table_name = table,
con = con,
schema = schema,
chunksize = chunk, )
if chunk:
exist = pd.concat(exist, ignore_index=True)
return df
def give_z(x):
if x.type == 'Polygon':
x = [x]
zlist = []
for polygon in x:
zlist.extend([c[-1] for c in polygon.exterior.coords[:-1]])
for inner_ring in polygon.interiors:
zlist.extend([c[-1] for c in inner_ring.coords[:-1]])
return zlist
#return sum(zlist)/len(zlist) #In your case to get mean. Or just return zlist[0] if they are all the same
def get_data(table,schema,engine=con_bdcen,chunk=chunk):
tmp = pd.read_sql_table(
table_name = table,
# sql = sql,
con = engine,
# geom_col = 'geom',
schema = schema,
# crs = crs,
chunksize = chunk, )
# Mise en forme des données
# start_time = dt.datetime.today()
if chunk:
df = pd.concat(tmp, ignore_index=True)
else:
df = tmp.copy()
if 'geom' in df.columns and not df[~df['geom'].isna()].empty:
df = gpd.GeoDataFrame(df)
df['geom'] = df.geom.apply(lambda x: shape.to_shape(x))
df.set_geometry('geom', inplace=True, crs=crs)
return df
# Initiation des connexions
# con_bdcen = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user_in,pwd_in,adr_in,port_in,base_in), echo=False)
# con_in = engine_in.connect()
# engine_out = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user_out,pwd_out,adr_out,port_out,base_out), echo=False)
# con_fon = engine_out.connect()
########################
###### MAIN ######
########################
# get table milieu
tmi = get_data(table = 'd_milieux', schema='sites', engine=con_fon)
# Import data
print('INIT import data ........... %s sec'%( time_exec(start_time) ))
sql = "SELECT * FROM {0}.{1}".format(schema_in, table_in)
df = get_data(table = table_in, schema=schema_in, engine=con_bdcen)
df.loc[(df.code_site=='RNIP')&(df.type_zonage=='ZO'),'nom_site'] = df.loc[(df.code_site=='RNIP')&(df.type_zonage=='ZI'),'nom_site'].values[0]
df = df[df.date_fin.isna()]
df.sort_values('code_site',inplace=True)
# Table sites
tab_out = 'sites'
name_col_out = con_fon.dialect.get_columns(con_fon, tab_out, schema=schema_out)
name_col_out = [ sub['name'] for sub in name_col_out ]
dictio = {
'code_site': 'site_id',
'nom_site' : 'site_nom',
# 'id_mnhn' : 'mnhn_id',
# 'idfcen' : 'fcen_id',
'milieux': 'milieu_id',
'date_ajout': 'annee_creation',
'surface_ha': 'surf_m2',
}
df_sites = gpd.GeoDataFrame(df[dictio.keys()], geometry=df.geom, crs=crs)
df_sites.rename(columns= dictio, inplace=True)
df_sites.rename(columns= {'geometry': 'geom'}, inplace=True)
df_sites.sort_values(['site_id','surf_m2'],inplace=True)
df_sites.drop_duplicates(subset=['site_id','site_nom','surf_m2'],inplace=True)
df_sites.loc[df_sites.site_nom.duplicated(keep='first'),'site_id'] = df_sites.loc[df_sites.site_nom.duplicated(keep='first'),'site_id'] + '_ZO'
df_sites.loc[df_sites.site_nom.duplicated(keep='last'),'site_id'] = df_sites.loc[df_sites.site_nom.duplicated(keep='last'),'site_id'] + '_ZI'
df_sites.loc[df_sites.site_id.str.endswith('ZO'),'site_nom'] = df_sites.loc[df_sites.site_id.str.endswith('ZO'),'site_nom'] + ' - ZO'
df_sites.loc[df_sites.site_id.str.endswith('ZI'),'site_nom'] = df_sites.loc[df_sites.site_id.str.endswith('ZI'),'site_nom'] + ' - ZI'
# Correspondance site / type_site
site_gere = '|'.join(df[df.gestion].code_site.unique().tolist())
site_asst = '|'.join(df[df.assist_cnv|df.assist_col].code_site.unique().tolist())
df_sites['typsite_id'] = '0'
df_sites.loc[df_sites.site_id.str.contains(site_asst), 'typsite_id'] = '2-1'
df_sites.loc[df_sites.site_id.str.contains(site_gere), 'typsite_id'] = '1-1'
# Correspondance site / milieu_lib_simpl
df_sites.loc[df_sites['milieu_id'] == 'Tourbières et marais', 'milieu_id'] = 'Milieux humides'
df_sites.loc[df_sites['milieu_id'] == 'Gîte à chiroptères', 'milieu_id'] = 'Gîtes à chiroptères'
df_sites.loc[df_sites['milieu_id'] == 'Ecosystèmes alluviaux', 'milieu_id'] = 'Milieux alluviaux'
df_sites.loc[df_sites['milieu_id'] == 'Ecosystèmes aquatiques', 'milieu_id'] = 'Milieux aquatiques'
df_sites.loc[df_sites['milieu_id'] == 'Pelouses sèches', 'milieu_id'] = 'Pelouses sèches'
df_sites.loc[df_sites['milieu_id'].isna(), 'milieu_id'] = 'N.P.'
for m in df_sites['milieu_id'].unique():
df_sites.loc[df_sites['milieu_id'] == m, 'milieu_id'] = tmi[
(tmi['milieu_lib_simpl'] == m) |
(tmi['milieu_lib'] == m )
]['milieu_id'].values[0]
# Complétion des champs pour test...
# df_sites['annee_creation'] = '2021'
df_sites.loc[~df_sites.annee_creation.isna(),'annee_creation'] = df_sites.loc[
~df_sites.annee_creation.isna()
,'annee_creation'].astype(str).str[:4]
df_sites['annee_creation'].fillna(9999,inplace=True)
if all(df_sites['geom'].has_z):
# Suppression de la dimension Z
geom_type = df_sites['geom'].geom_type
df_sites['geom'] = [wkb.loads(wkb.dumps(geom, output_dimension=2)) for geom in df_sites['geom']]
df_sites.set_geometry('geom', drop=False, inplace=True, crs=crs)
# df_sites.rename(columns= {'geometry': 'geom'}, inplace=True)
df_sites.to_postgis(
name = tab_out,
con = con_fon,
schema = schema_out,
index = False,
if_exists = 'append',
chunksize = chunk,
geom_col = 'geom',
)
# Table d_typsite
# Table d_milieux
# print('INIT construction data for d_milieux table ...... %s sec'%( time_exec(start_time) ))
# df = pd.DataFrame(DF['tmilieux'])
# df['milieu_lib_simpl'] = None
# df['milieu_descrip'] = None
# df.drop_duplicates(inplace=True)
# df.dropna(inplace=True, how='all',axis=0)
# df = drop_exist(df, con=con_fon, table='d_milieux', schema=schema_out)
# sql = "UPDATE {0}.{1} SET tmilieux = 'Ecosystèmes alluviaux' WHERE tmilieux = 'Ecosystemes alluviaux'".format(schema_in, table_in)
# print(sql)
# con_in.execute(sql)
# for index, d in df.iterrows():
# tmp = df[df.id != d.id].copy()
# tmp['intersect'] = tmp.geometry.intersects(d.geom)
# if not tmp[tmp.intersect].empty:
# print(tmp[tmp.intersect])
# else :
# print('No insterction for {}'.format(d.id))
0_FONCIER/get_infos_parcelle.py
+18
View File
@@ -0,0 +1,18 @@
import pycen
import geopandas as gpd
par_id = '38412000ZD0136'
PATH_OUT = '/home/colas/Documents/tmp/'+par_id
sql = "SELECT par_id, geom FROM cadastre.parcelles_38 WHERE par_id = '%s'"%par_id
parc = gpd.read_postgis(sql,pycen.con_fon)
parc.to_crs(4326).to_file(PATH_OUT+'.geojson', driver='GeoJSON')
parc.to_crs(4326).to_file(PATH_OUT+'.gpkg', driver='GPKG')
sql = "SELECT * FROM zones_humides.v_zoneshumides WHERE ST_INTERSECTS(geom,'SRID=2154;%s')"%parc.geometry[0]
zh =gpd.read_postgis(sql,pycen.con)
sql = "SELECT * FROM saisie.vm_synthese_observations WHERE ST_INTERSECTS(geom,'SRID=2154;%s')"%parc.geometry[0]
obs_sicen =gpd.read_postgis(sql,pycen.con_sicen)
0_FONCIER/grant_table.py
+52
View File
@@ -0,0 +1,52 @@
from pycen import con_bdcen,con_cad,con_fon as con
import geopandas as gpd
from sqlalchemy import create_engine, text
def get_list_table(con,sch):
return con.dialect.get_table_names(con,schema=sch)
def get_list_view(con,sch):
return con.dialect.get_view_names(con,schema=sch)
def get_table_pkey(con,sch,tab):
return con.dialect.get_foreign_keys(con,tab,sch)['constrained_columns']
def grant_all_table(con,sch):
# sch = 'cadastre'
sql = 'GRANT USAGE ON SCHEMA "%s" TO grp_consult;'%sch
with con.begin() as cnx:
cnx.execute(sql)
print('GRANT USAGE TO grp_consult FOR SCHEMA "%s"'%(sch))
for tab in get_list_table(con,sch):
sql = 'GRANT SELECT ON TABLE "%s".%s TO grp_consult;'%(sch,tab)
with con.begin() as cnx:
cnx.execute(sql)
print('GRANT SELECT TO grp_consult FOR "%s".%s'%(sch,tab))
for tab in get_list_view(con,sch):
sql = 'GRANT SELECT ON TABLE "%s".%s TO grp_consult;'%(sch,tab)
with con.begin() as cnx:
cnx.execute(sql)
print('GRANT SELECT TO grp_consult FOR "%s".%s'%(sch,tab))
def revoke_all_table(con,sch):
for tab in get_list_table(con,sch):
sql = 'REVOKE SELECT ON "%s".%s FROM grp_consult;'%(sch,tab)
with con.begin() as cnx:
cnx.execute(sql)
print('REVOKE SELECT grp_consult FOR "%s".%s'%(sch,tab))
for tab in get_list_view(con,sch):
sql = 'REVOKE SELECT ON "%s".%s FROM grp_consult;'%(sch,tab)
with con.begin() as cnx:
cnx.execute(sql)
print('REVOKE SELECT TO grp_consult FOR "%s".%s'%(sch,tab))
if __name__ == "__main__":
sch_cad = '42_202603'
sch_old = '42_202501'
grant_all_table(con_cad,sch_cad)
# revoke_all_table(con_cad,sch_old)
0_FONCIER/new_sites.py
+36
View File
@@ -0,0 +1,36 @@
from pycen import con_fon, update_to_sql
import geopandas as gpd
from os import path
def milieu(x):
d_milieu = gpd.pd.read_sql('SELECT * FROM sites.d_milieux',con_fon)
dict_m = dict(zip(d_milieu.milieu_lib,d_milieu.milieu_id))
x.replace({'milieu':dict_m},inplace=True)
x.rename(columns={'milieu':'milieu_id'},inplace=True)
def typ_site(x):
d_site = gpd.pd.read_sql('SELECT * FROM sites.d_typsite',con_fon)
dict_s = dict(zip(d_site.typsite_lib,d_site.typsite_id))
x.replace({'typ_site':dict_s},inplace=True)
x.rename(columns={'typ_site':'typsite_id'},inplace=True)
if __name__ == "__main__":
PATH = '/home/colas/Documents/tmp/Foncier/LONE'
file = 'LONE.xlsx'
geofile = 'Contour_site_lônes.shp'
df = (gpd.pd.read_excel(path.join(PATH,file))
.rename(columns={'id_site':'site_id'}))
milieu(df)
typ_site(df)
_gdf = gpd.read_file(path.join(PATH,geofile))
gdf = gpd.GeoDataFrame(df,geometry=[_gdf.unary_union],crs=_gdf.crs)
gdf.rename_geometry('geom',inplace=True)
gdf['surf_m2'] = gdf.area
gdf.to_postgis('sites',con_fon,'sites',if_exists='append',index=False)
gdf.rename_geometry('geom_ecolo',inplace=True)
update_to_sql(gdf,con_fon,'sites','sites','site_id')
0_FONCIER/remove_parcelles_site.py
+39
View File
@@ -0,0 +1,39 @@
from pycen import con_fon
import geopandas as gpd
import sys
site_id = 'CRAS'
lst_parcelle = None
site = gpd.read_postgis(
"SELECT * FROM sites.v_sites WHERE site_id = '%s'"%site_id,
con_fon)
parc = gpd.read_postgis(
"SELECT * FROM sites.v_sites_parcelles WHERE site_id = '%s'"%site_id,
con_fon)
# lst_parcelle = parc[~parc.intersects(site.unary_union)].par_id.unique().tolist()
lst_parcelle = parc.par_id.unique().tolist()
for par_id in lst_parcelle:
try:
with con_fon.begin() as cnx:
cnx.execute("SELECT cadastre.suppr_parcelles_cen('{par_id}','{site_id}')"\
.format(
par_id=par_id,site_id=site_id
)
)
except Exception as e:
print(e)
print('\n')
print('La parcelle %s ne peut pas être supprimé'%par_id)
with con_fon.begin() as cnx:
r = cnx.execute("SELECT admin_sig.refresh_mview_foncier('{site_id}')"\
.format(
site_id=site_id
)
)
res = r.fetchall()
0_FONCIER/v_veillefonciere.py
+27
View File
@@ -0,0 +1,27 @@
from sqlalchemy import text
from pycen import con_fon
v_veillefonciere = """
DROP VIEW IF EXISTS _tdb_f6cen.v_veillefonciere;
CREATE OR REPLACE VIEW _tdb_f6cen.v_veillefonciere
AS
SELECT
row_number() OVER (ORDER BY SPLIT_PART(site_id,'_',1) ASC) AS gid,
SPLIT_PART(site_id,'_',1) AS site_id,
SPLIT_PART(site_nom,' - Z',1) AS site_nom,
enjeu_lib,
'CONSERVATOIRE D''ESPACES NATURELS ISERE'::text AS cen,
38::int AS dept,
st_area(st_union(geom_vf)) AS superficie,
st_union(geom_vf) geom
FROM sites.sites
JOIN sites.d_enjeux USING (enjeu_id)
WHERE veillefonciere
GROUP BY 2,3,4
ORDER BY 1
;
"""
with con_fon.begin() as cnx:
cnx.execute(text(v_veillefonciere))
10_PAQ/1C-2-1.py
+97
View File
File diff suppressed because one or more lines are too long
11_CALC_RHOMEO/insert_bdtopo3.py
+173
View File
@@ -0,0 +1,173 @@
from sqlalchemy import create_engine, text,PrimaryKeyConstraint
from geoalchemy2 import Geometry
import geopandas as gpd
from os import listdir,path
from pycen import con_rho as con
# user_cad = 'xxxxxxx' # utilisateur de connexion à la bdd
# pwd_cad = 'xxxxxxx' # mot de passe de la bdd
# adr_cad = '91.134.194.221' # adresse ip de la bdd
# port_cad = '5432' # port de la bdd
# base_cad = 'rhomeo' # nom de la bdd
schema_cad = 'bdtopo3'
# con = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user_cad,pwd_cad,adr_cad,port_cad,base_cad), echo=False)
def get_list_table(con,sch="public"):
with con.begin() as cnx:
return cnx.dialect.get_table_names(cnx,schema=sch)
def get_columns(con,sch,tab):
with con.begin() as cnx:
return cnx.dialect.get_columns(cnx,tab,sch)
def get_table_pkey(con,sch,tab):
with con.begin() as cnx:
return cnx.dialect.get_foreign_keys(cnx,tab,sch)
# for tab in ['spatial_ref_sys',#'info_metadonnees','metadonnees_lot','metadonnees_theme',
# 'layer_styles']:
# sql = """ALTER TABLE bdtopo3.%s
# SET SCHEMA public;"""%tab
# with con.begin() as cnx:
# cnx.execute(sql)
# print('Table %s SET TO SCHEMA bdtopo3'%(tab))
lst_zip = [
'/vsizip//home/colas/Craig/CEN_38/BD Vectorielles/IGN - BDTOPO - 202309/bdtopo_dep07_202309_geopackage.zip/BDT_3-3_GPKG_LAMB93_D007-ED2023-09-15/BDT_3-3_GPKG_LAMB93_D007-ED2023-09-15.gpkg',
'/vsizip//home/colas/Craig/CEN_38/BD Vectorielles/IGN - BDTOPO - 202309/bdtopo_dep26_202309_geopackage.zip/BDT_3-3_GPKG_LAMB93_D026-ED2023-09-15/BDT_3-3_GPKG_LAMB93_D026-ED2023-09-15.gpkg',
'/vsizip//home/colas/Craig/CEN_38/BD Vectorielles/IGN - BDTOPO - 202309/bdtopo_dep42_202309_geopackage.zip/BDT_3-3_GPKG_LAMB93_D042-ED2023-09-15/BDT_3-3_GPKG_LAMB93_D042-ED2023-09-15.gpkg'
]
lst_tab = [
'cimetiere', # BATI
'construction_surfacique', # BATI
'reservoir',
'batiment',
'troncon_de_route',
'piste_d_aerodrome',
'troncon_de_voie_ferree',
'terrain_de_sport',
]
# cimetiere
# construction_surfacique
# reservoir
# batiment
# terrain_de_sport
dict_cols = {
'id':'cleabs'
}
PATH = "/home/colas/Documents/5_BDD/6_IGN/BDTOPO/BDTOPO_3-3_TOUSTHEMES_SHP_LAMB93_D074_2023-09-15/BDTOPO/1_DONNEES_LIVRAISON_2023-09-00196/BDT_3-3_SHP_LAMB93_D074-ED2023-09-15/BATI"
PATH2 = '../TRANSPORT'
for tab in lst_tab:
file = tab.upper()+'.shp'
if file in listdir(path.join(PATH)):
df = gpd.read_file(path.join(PATH,file))
df.columns = df.columns.str.lower()
df.set_index('id',inplace=True)
# df.rename(columns=dict_cols,inplace=True)
print('Load %s'%tab)
else: continue
# columns_type = get_columns(con,schema_cad,tab)
# geom_type = [x for x in columns_type if x['name']=='geometrie']
# print('Ident geom_type OK')
# if geom_type:
# geom_type = geom_type[0]
# if df.geometry.name != geom_type['name']:
# df.rename_geometry(geom_type['name'],inplace=True)
if df.geometry.name != 'geom':
df.rename_geometry('geom',inplace=True)
has_z = True if df.has_z.all() else False
unique_geom = df.geom_type.unique()
geom_type = df.geom_type[0].upper() if len(unique_geom) == 1 else 'GEOMETRY'
geom_type = geom_type+'Z'if has_z else geom_type
(df
.to_wkt()
.to_sql(
tab,
con,
schema_cad,
if_exists='append',
index=True,
dtype={
'geom':Geometry(geometry_type=geom_type,srid=2154)
}
)
)
print(tab)
# get_table_pkey(con,schema_cad,tab)
# get_columns(con,schema_cad,tab)
## 38
# Load troncon_de_voie_ferree
# Ident geom_type OK
# troncon_de_voie_ferree
# Load terrain_de_sport
# Ident geom_type OK
# terrain_de_sport
## 26
# Load cimetiere
# Ident geom_type OK
# cimetiere
# Load construction_surfacique
# Ident geom_type OK
# construction_surfacique
# Load reservoir
# Ident geom_type OK
# reservoir
# Load batiment
# Ident geom_type OK
# batiment
## NO INSERT
# Load troncon_de_route
# Ident geom_type OK
for Zip in lst_zip:
print('\n'+Zip+'\n')
for tab in get_list_table(con,schema_cad):
if tab not in lst_tab:
continue
if Zip in lst_zip[:2] or (
Zip==lst_zip[2] and tab in [
'cimetiere',
'construction_surfacique',
'reservoir',
'batiment',
]):
continue
df = gpd.read_file(Zip,layer=tab)
print('Load %s'%tab)
columns_type = get_columns(con,schema_cad,tab)
geom_type = [x for x in columns_type if x['name']=='geometrie']
print('Ident geom_type OK')
if geom_type:
geom_type = geom_type[0]
if df.geometry.name != geom_type['name']:
df.rename_geometry(geom_type['name'],inplace=True)
df.to_wkt().to_sql(
tab,
con,
schema_cad,
if_exists='append',
index=False,
dtype={geom_type['name']:geom_type['type']} if geom_type else None
)
print(tab)
11_CALC_RHOMEO/insert_bvmdo.py
+40
View File
@@ -0,0 +1,40 @@
from geoalchemy2 import Geometry
import geopandas as gpd
from sqlalchemy import create_engine
user_cad = 'cgeier' # utilisateur de connexion à la bdd
pwd_cad = 'adm1n*bdCen' # mot de passe de la bdd
adr_cad = '91.134.194.221' # adresse ip de la bdd
port_cad = '5432' # port de la bdd
base_cad = 'rhomeo' # nom de la bdd
schema_cad = 'refgeo'
con = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user_cad,pwd_cad,adr_cad,port_cad,base_cad), echo=False)
file = '/home/colas/Documents/5_BDD/BASSIN_VERSANT/AGENCE EAU/bvmdo/bvmdo.shp'
df = gpd.read_file(file)
df.rename_geometry('geom', inplace=True)
has_z = True if df.has_z.all() else False
unique_geom = df.geom_type.unique()
geom_type = df.geom_type[0].upper() if len(unique_geom) == 1 else 'GEOMETRY'
geom_type = geom_type+'Z'if has_z else geom_type
(df
.to_wkt()
.to_sql(
'zh',
con,
schema_cad,
if_exists='append',
index=False,
dtype={
'geom':Geometry(geometry_type=geom_type,srid=2154)
}
)
)
sql = 'SELECT * FROM tachenew.tache_urbaine'
sql = 'SELECT * FROM arvfnew.arvf_global_buffer'
df = gpd.read_postgis(sql,con)
11_CALC_RHOMEO/run_indicRhomeo.py
+369
View File
@@ -0,0 +1,369 @@
from pycen import con_rho
def indicI12_zh(con):
sql = """
DROP TABLE IF EXISTS indicnew.indic_sig_zh;
CREATE TABLE IF NOT EXISTS indicnew.indic_sig_zh AS (
SELECT DISTINCT
zh.id_bdd,
st_area2d(zh.geom)::double precision AS area
FROM refgeo.zh
);
ALTER TABLE indicnew.indic_sig_zh
ADD CONSTRAINT indicnew_sig_zh_pkey PRIMARY KEY (id_bdd);
ALTER TABLE indicnew.indic_sig_zh
ADD COLUMN IF NOT EXISTS isole double precision;
ALTER TABLE indicnew.indic_sig_zh
ADD COLUMN IF NOT EXISTS diffus double precision;
ALTER TABLE indicnew.indic_sig_zh
ADD COLUMN IF NOT EXISTS groupe double precision;
ALTER TABLE indicnew.indic_sig_zh
ADD COLUMN IF NOT EXISTS peudense double precision;
ALTER TABLE indicnew.indic_sig_zh
ADD COLUMN IF NOT EXISTS dense double precision;
ALTER TABLE indicnew.indic_sig_zh
ADD COLUMN IF NOT EXISTS tresdense double precision;
--DROP TABLE IF EXISTS indicnew.indic_sig_zh_temp;
CREATE TABLE IF NOT EXISTS indicnew.indic_sig_zh_temp AS (
SELECT
zh.id_bdd,
tache_urbaine.reclasse AS reclasse,
Sum(st_area2d(st_intersection(tache_urbaine.geom, zh.geom)))::double precision AS reclasse_surf
FROM tachenew.tache_urbaine
JOIN refgeo.zh ON (st_intersects(tache_urbaine.geom, zh.geom))
GROUP BY zh.id_bdd, tache_urbaine.reclasse, zh.geom
);
UPDATE indicnew.indic_sig_zh SET isole = (SELECT indic_sig_zh_temp.reclasse_surf FROM indicnew.indic_sig_zh_temp WHERE indic_sig_zh.id_bdd = indic_sig_zh_temp.id_bdd AND indic_sig_zh_temp.reclasse = '1- isolé');
UPDATE indicnew.indic_sig_zh SET diffus = (SELECT indic_sig_zh_temp.reclasse_surf FROM indicnew.indic_sig_zh_temp WHERE indic_sig_zh.id_bdd = indic_sig_zh_temp.id_bdd AND indic_sig_zh_temp.reclasse = '2- diffus');
UPDATE indicnew.indic_sig_zh SET groupe = (SELECT indic_sig_zh_temp.reclasse_surf FROM indicnew.indic_sig_zh_temp WHERE indic_sig_zh.id_bdd = indic_sig_zh_temp.id_bdd AND indic_sig_zh_temp.reclasse = '3- groupé');
UPDATE indicnew.indic_sig_zh SET peudense = (SELECT indic_sig_zh_temp.reclasse_surf FROM indicnew.indic_sig_zh_temp WHERE indic_sig_zh.id_bdd = indic_sig_zh_temp.id_bdd AND indic_sig_zh_temp.reclasse = '4- urbain peu dense');
UPDATE indicnew.indic_sig_zh SET dense = (SELECT indic_sig_zh_temp.reclasse_surf FROM indicnew.indic_sig_zh_temp WHERE indic_sig_zh.id_bdd = indic_sig_zh_temp.id_bdd AND indic_sig_zh_temp.reclasse = '5- urbain dense');
UPDATE indicnew.indic_sig_zh SET tresdense = (SELECT indic_sig_zh_temp.reclasse_surf FROM indicnew.indic_sig_zh_temp WHERE indic_sig_zh.id_bdd = indic_sig_zh_temp.id_bdd AND indic_sig_zh_temp.reclasse = '6- urbain très dense');
DROP TABLE IF EXISTS indicnew.indic_sig_zh_temp;
ALTER TABLE indicnew.indic_sig_zh
ADD COLUMN IF NOT EXISTS bati_area double precision;
WITH t2 AS (
SELECT
t1.id_bdd,
t1.surf
FROM refgeo.zh
JOIN (
SELECT
zh.id_bdd,
Sum(st_area2d(st_intersection(zh.geom, tache_urbaine.geom))) AS surf
FROM refgeo.zh
JOIN tachenew.tache_urbaine ON (st_intersects(zh.geom, tache_urbaine.geom))
GROUP BY id_bdd
) AS t1 USING (id_bdd)
)
UPDATE indicnew.indic_sig_zh SET bati_area = t2.surf FROM t2 WHERE indic_sig_zh.id_bdd = t2.id_bdd;
ALTER TABLE indicnew.indic_sig_zh
ADD COLUMN IF NOT EXISTS arvf_area double precision;
WITH t2 AS (
SELECT
t1.id_bdd,
t1.surf
FROM refgeo.zh
JOIN (
SELECT
zh.id_bdd,
Sum(st_area2d(st_intersection(zh.geom, arvf_global_buffer.geom))) AS surf
FROM refgeo.zh
JOIN arvfnew.arvf_global_buffer ON (st_intersects(zh.geom, arvf_global_buffer.geom))
GROUP BY id_bdd
) AS t1 USING (id_bdd)
)
UPDATE indicnew.indic_sig_zh SET arvf_area = t2.surf FROM t2 WHERE indic_sig_zh.id_bdd = t2.id_bdd;
ALTER TABLE indicnew.indic_sig_zh
ADD COLUMN IF NOT EXISTS artif_area double precision;
WITH t2 AS (
SELECT
t1.id_bdd,
t1.surf
FROM refgeo.zh
JOIN (
SELECT
zh.id_bdd,
Sum(st_area2d(st_intersection(zh.geom, tache_artif.geom))) AS surf
FROM refgeo.zh
JOIN tachenew.tache_artif ON (st_intersects(zh.geom, tache_artif.geom))
GROUP BY id_bdd
) AS t1 USING (id_bdd)
)
UPDATE indicnew.indic_sig_zh SET artif_area = t2.surf FROM t2 WHERE indic_sig_zh.id_bdd = t2.id_bdd;
"""
with con.begin() as cnx:
cnx.execute(sql)
print('indicI12_zh created')
def indicI12_zh_buff(con):
sql = '''
ALTER TABLE indicnew.indic_sig_zh
ADD COLUMN IF NOT EXISTS area_buff double precision;
UPDATE indicnew.indic_sig_zh SET area_buff = st_area2d(zh.geom_buff) FROM refgeo.zh WHERE indic_sig_zh.id_bdd = zh.id_bdd;
ALTER TABLE indicnew.indic_sig_zh
ADD COLUMN IF NOT EXISTS isole_buff double precision;
ALTER TABLE indicnew.indic_sig_zh
ADD COLUMN IF NOT EXISTS diffus_buff double precision;
ALTER TABLE indicnew.indic_sig_zh
ADD COLUMN IF NOT EXISTS groupe_buff double precision;
ALTER TABLE indicnew.indic_sig_zh
ADD COLUMN IF NOT EXISTS peudense_buff double precision;
ALTER TABLE indicnew.indic_sig_zh
ADD COLUMN IF NOT EXISTS dense_buff double precision;
ALTER TABLE indicnew.indic_sig_zh
ADD COLUMN IF NOT EXISTS tresdense_buff double precision;
--DROP TABLE IF EXISTS indicnew.indic_sig_zh_temp;
CREATE TABLE IF NOT EXISTS indicnew.indic_sig_zh_temp AS (
SELECT
zh.id_bdd,
tache_urbaine.reclasse AS reclasse,
Sum(st_area2d(st_Intersection(tache_urbaine.geom, zh.geom_buff)))::double precision AS reclasse_surf
FROM tachenew.tache_urbaine
JOIN refgeo.zh ON (st_intersects(tache_urbaine.geom, zh.geom_buff))
GROUP BY zh.id_bdd, tache_urbaine.reclasse, zh.geom_buff
);
UPDATE indicnew.indic_sig_zh SET isole_buff = (SELECT indic_sig_zh_temp.reclasse_surf FROM indicnew.indic_sig_zh_temp WHERE indic_sig_zh.id_bdd = indic_sig_zh_temp.id_bdd AND indic_sig_zh_temp.reclasse = '1- isolé');
UPDATE indicnew.indic_sig_zh SET diffus_buff = (SELECT indic_sig_zh_temp.reclasse_surf FROM indicnew.indic_sig_zh_temp WHERE indic_sig_zh.id_bdd = indic_sig_zh_temp.id_bdd AND indic_sig_zh_temp.reclasse = '2- diffus');
UPDATE indicnew.indic_sig_zh SET groupe_buff = (SELECT indic_sig_zh_temp.reclasse_surf FROM indicnew.indic_sig_zh_temp WHERE indic_sig_zh.id_bdd = indic_sig_zh_temp.id_bdd AND indic_sig_zh_temp.reclasse = '3- groupé');
UPDATE indicnew.indic_sig_zh SET peudense_buff = (SELECT indic_sig_zh_temp.reclasse_surf FROM indicnew.indic_sig_zh_temp WHERE indic_sig_zh.id_bdd = indic_sig_zh_temp.id_bdd AND indic_sig_zh_temp.reclasse = '4- urbain peu dense');
UPDATE indicnew.indic_sig_zh SET dense_buff = (SELECT indic_sig_zh_temp.reclasse_surf FROM indicnew.indic_sig_zh_temp WHERE indic_sig_zh.id_bdd = indic_sig_zh_temp.id_bdd AND indic_sig_zh_temp.reclasse = '5- urbain dense');
UPDATE indicnew.indic_sig_zh SET tresdense_buff = (SELECT indic_sig_zh_temp.reclasse_surf FROM indicnew.indic_sig_zh_temp WHERE indic_sig_zh.id_bdd = indic_sig_zh_temp.id_bdd AND indic_sig_zh_temp.reclasse = '6- urbain très dense');
DROP TABLE IF EXISTS indicnew.indic_sig_zh_temp;
ALTER TABLE indicnew.indic_sig_zh
ADD COLUMN IF NOT EXISTS bati_area_buff double precision;
WITH t2 AS (
SELECT
t1.id_bdd,
t1.surf
FROM refgeo.zh
JOIN (
SELECT
zh.id_bdd,
Sum(st_area2d(st_Intersection(zh.geom_buff, tache_urbaine.geom))) AS surf
FROM refgeo.zh
JOIN tachenew.tache_urbaine ON (st_intersects(zh.geom_buff, tache_urbaine.geom))
GROUP BY id_bdd
) AS t1 USING (id_bdd)
)
UPDATE indicnew.indic_sig_zh SET bati_area_buff = t2.surf FROM t2 WHERE indic_sig_zh.id_bdd = t2.id_bdd;
ALTER TABLE indicnew.indic_sig_zh
ADD COLUMN IF NOT EXISTS arvf_area_buff double precision;
WITH t2 AS (
SELECT
t1.id_bdd,
t1.surf
FROM refgeo.zh
JOIN (
SELECT
zh.id_bdd,
Sum(st_area2d(st_intersection(zh.geom_buff, arvf_global_buffer.geom))) AS surf
FROM refgeo.zh
JOIN arvfnew.arvf_global_buffer ON (st_intersects(zh.geom_buff, arvf_global_buffer.geom))
GROUP BY id_bdd
) AS t1 USING (id_bdd)
)
UPDATE indicnew.indic_sig_zh SET arvf_area_buff = t2.surf FROM t2 WHERE indic_sig_zh.id_bdd = t2.id_bdd;
ALTER TABLE indicnew.indic_sig_zh
ADD COLUMN IF NOT EXISTS artif_area_buff double precision;
WITH t2 AS (
SELECT
t1.id_bdd,
t1.surf
FROM refgeo.zh
JOIN (
SELECT
zh.id_bdd,
Sum(st_area2d(st_intersection(zh.geom_buff, tache_artif.geom))) AS surf
FROM refgeo.zh
JOIN tachenew.tache_artif ON (st_intersects(zh.geom_buff, tache_artif.geom))
GROUP BY id_bdd
) AS t1 USING (id_bdd)
)
UPDATE indicnew.indic_sig_zh SET artif_area_buff = t2.surf FROM t2 WHERE indic_sig_zh.id_bdd = t2.id_bdd;
'''
with con.begin() as cnx:
cnx.execute(sql)
print('indicI12_zh_buff created')
def indicI13_zh(con):
sql = """
-- Pression agricole sur sites
ALTER TABLE IF EXISTS indicnew.indic_sig_zh
DROP COLUMN IF EXISTS agri_ign_impact_area;
ALTER TABLE indicnew.indic_sig_zh
ADD COLUMN IF NOT EXISTS agri_ign_impact_area double precision;
WITH calcul_impact_rpg AS (
SELECT
ilots_sites.id_bdd,
Sum(ilots_sites.surf_impact) AS surf_impact,
Sum(ilots_sites.surf_impact) * 100 / ilots_sites.surf_site AS pourc_impact
FROM (
SELECT
t1.num_ilot,
id_bdd,
st_area(t2.geom) AS surf_site,
st_area(st_intersection(t1.geom, t2.geom)) AS surf_inters,
(st_area(st_intersection(t1.geom, t2.geom))) * t1.pourc_impact AS surf_impact,
st_intersection(t1.geom, t2.geom) AS geom
FROM rpgnew.rpg_global t1
JOIN refgeo.zh t2 ON (st_intersects(t1.geom, t2.geom))
) AS ilots_sites
GROUP BY ilots_sites.id_bdd, ilots_sites.surf_site
)
UPDATE indicnew.indic_sig_zh SET agri_ign_impact_area = calcul_impact_rpg.surf_impact FROM calcul_impact_rpg WHERE indic_sig_zh.id_bdd = calcul_impact_rpg.id_bdd;
-- Requête exécutée avec succès en 2 s 116 msec.
"""
with con.begin() as cnx:
cnx.execute(sql)
print('indicI13_zh created')
def indicI13_zh_buff(con):
sql = """
-- Pression agricole sur sites_buffer
ALTER TABLE IF EXISTS indicnew.indic_sig_zh
DROP COLUMN IF EXISTS agri_ign_impact_area_buff;
ALTER TABLE indicnew.indic_sig_zh
ADD COLUMN agri_ign_impact_area_buff double precision;
WITH calcul_impact_rpg_buff AS (
SELECT
ilots_sites.id_bdd,
Sum(ilots_sites.surf_impact_buff) AS surf_impact_buff,
Sum(ilots_sites.surf_impact_buff) * 100 / ilots_sites.surf_site_buff AS pourc_impact_buff
FROM (
SELECT
t1.num_ilot,
t2.id_bdd,
st_area(t2.geom_buff) AS surf_site_buff,
st_area(st_intersection(t1.geom, t2.geom_buff)) AS surf_inters,
(st_area(st_intersection(t1.geom, t2.geom_buff))) * t1.pourc_impact AS surf_impact_buff,
st_intersection(t1.geom, t2.geom_buff) AS geom
FROM rpgnew.rpg_global t1
JOIN refgeo.zh t2 ON (st_intersects(t1.geom, t2.geom_buff))
) AS ilots_sites
GROUP BY ilots_sites.id_bdd, ilots_sites.surf_site_buff
)
UPDATE indicnew.indic_sig_zh SET agri_ign_impact_area_buff = calcul_impact_rpg_buff.surf_impact_buff FROM calcul_impact_rpg_buff WHERE indic_sig_zh.id_bdd = calcul_impact_rpg_buff.id_bdd;
-- Requête exécutée avec succès en 4 s 843 msec.
"""
with con.begin() as cnx:
cnx.execute(sql)
print('indicI13_zh_buff created')
def indicI13_bv(con):
sql = """
-- Pression agricole sur BVMDO
ALTER TABLE IF EXISTS indicnew.indic_sig_bvmdo
DROP COLUMN IF EXISTS agri_ign_impact_area;
ALTER TABLE indicnew.indic_sig_bvmdo
ADD COLUMN agri_ign_impact_area double precision;
WITH calcul_impact_rpg_bvmdo AS (
SELECT
ilots_bvmdo.cmdo,
Sum(ilots_bvmdo.surf_impact) AS surf_impact,
Sum(ilots_bvmdo.surf_impact) * 100 / ilots_bvmdo.surf_bvmdo AS pourc_impact
FROM (
SELECT
t1.num_ilot,
t2.cmdo,
st_area(t2.geom) AS surf_bvmdo,
st_area(st_intersection(t1.geom, t2.geom)) AS surf_inters,
(st_area(st_intersection(t1.geom, t2.geom))) * t1.pourc_impact AS surf_impact,
st_intersection(t1.geom, t2.geom) AS geom
FROM rpgnew.rpg_global t1
JOIN refgeo.bvmdo t2 ON (st_intersects(t1.geom, t2.geom))
) AS ilots_bvmdo
GROUP BY ilots_bvmdo.cmdo, ilots_bvmdo.surf_bvmdo
)
UPDATE indicnew.indic_sig_bvmdo SET agri_ign_impact_area = calcul_impact_rpg_bvmdo.surf_impact FROM calcul_impact_rpg_bvmdo WHERE indic_sig_bvmdo.cmdo = calcul_impact_rpg_bvmdo.cmdo;
-- Requête exécutée avec succès en 6 s 414 msec.
"""
with con.begin() as cnx:
cnx.execute(sql)
print('indicI13_bv created')
def create_zhGeom_buff(con):
sql = """
ALTER TABLE refgeo.zh ADD COLUMN IF NOT EXISTS geom_buff geometry(MultiPolygon,2154);
UPDATE refgeo.zh
SET geom_buff = st_multi(st_difference(st_buffer(zh.geom, ( |/ (st_area(zh.geom)*2/pi()) ) - ( |/ (st_area(zh.geom)/pi()) ), 40), zh.geom));
"""
with con.begin() as cnx:
cnx.execute(sql)
print('geom_buff created')
def indicBilan(con):
sql = """
DROP TABLE IF EXISTS indicnew.indic_bilan_zh;
CREATE TABLE indicnew.indic_bilan_zh AS
WITH
t1 AS (
SELECT zh.id_bdd as id_bdd, Count(*) as nb_zone_hydro, Sum(indic_sig_bvmdo.area) as surf_zone_hydro, Sum(indic_sig_bvmdo.isole) as isole_zone_hydro, Sum(indic_sig_bvmdo.diffus) as diffus_zone_hydro, Sum(indic_sig_bvmdo.groupe) as groupe_zone_hydro, Sum(indic_sig_bvmdo.peudense) as peudense_zone_hydro, Sum(indic_sig_bvmdo.dense) as dense_zone_hydro, Sum(indic_sig_bvmdo.tresdense) as tresdense_zone_hydro, Sum(indic_sig_bvmdo.bati_area) as surf_zone_hydro_bati, Sum(indic_sig_bvmdo.arvf_area) as surf_zone_hydro_arvf, Sum(indic_sig_bvmdo.artif_area) as surf_zone_hydro_artif, Sum(indic_sig_bvmdo.agri_ign_impact_area) as surf_zone_hydro_agri
FROM refgeo.zh
JOIN refgeo.bvmdo ON (st_intersects(zh.geom, bvmdo.geom))
JOIN indicnew.indic_sig_bvmdo USING (cmdo)
GROUP BY zh.id_bdd)
SELECT
id_bdd,
coalesce(indic_sig_zh.area, 0) as surf_zh,
coalesce(indic_sig_zh.area, 0) +
coalesce(indic_sig_zh.area_buff, 0) as surf_rhomeo,
CASE WHEN (coalesce(indic_sig_zh.bati_area, 0) + coalesce(indic_sig_zh.bati_area_buff, 0)) > 0
THEN
(coalesce(indic_sig_zh.peudense, 0) +
coalesce(indic_sig_zh.dense, 0) +
coalesce(indic_sig_zh.tresdense, 0) +
coalesce(indic_sig_zh.peudense_buff, 0) +
coalesce(indic_sig_zh.dense_buff, 0) +
coalesce(indic_sig_zh.tresdense_buff, 0)) / (coalesce(indic_sig_zh.bati_area, 0) + coalesce(indic_sig_zh.bati_area_buff, 0)) * 100
ELSE
0
END as presdirect_urba,
(coalesce(indic_sig_zh.artif_area, 0) +
coalesce(indic_sig_zh.artif_area_buff, 0)) / (coalesce(indic_sig_zh.area, 0) + coalesce(indic_sig_zh.area_buff, 0)) * 100 as presdirect_artif,
(coalesce(indic_sig_zh.agri_ign_impact_area, 0) + coalesce(indic_sig_zh.agri_ign_impact_area_buff, 0)) / (indic_sig_zh.area + indic_sig_zh.area_buff) * 100 as presdirect_agri,
CASE WHEN t1.surf_zone_hydro_bati > 0 THEN
(coalesce(t1.peudense_zone_hydro, 0) +
coalesce(t1.dense_zone_hydro, 0) +
coalesce(t1.tresdense_zone_hydro, 0)) / t1.surf_zone_hydro_bati * 100
ELSE
0
END as presindir_urba,
coalesce(t1.surf_zone_hydro_artif, 0) / t1.surf_zone_hydro * 100 as presindir_artif,
coalesce(t1.surf_zone_hydro_agri, 0) / t1.surf_zone_hydro * 100 as presindir_agri,
LEFT(current_timestamp(0)::text, 10) as date_maj
FROM refgeo.zh
INNER JOIN indicnew.indic_sig_zh USING (id_bdd)
INNER JOIN t1 USING (id_bdd);
ALTER TABLE indicnew.indic_bilan_zh DROP CONSTRAINT IF EXISTS indic_bilan_zh_pkey;
ALTER TABLE indicnew.indic_bilan_zh
ADD CONSTRAINT indic_bilan_zh_pkey PRIMARY KEY(id_bdd);
"""
with con.begin() as cnx:
cnx.execute(sql)
print('indicBilan created')
if __name__ == '__main__':
from pycen import con_rho
create_zhGeom_buff(con_rho)
indicI12_zh(con_rho)
indicI12_zh_buff(con_rho)
indicI13_zh(con_rho)
indicI13_zh_buff(con_rho)
indicBilan(con_rho)
12_ODK/remunt.sh
+14
View File
@@ -0,0 +1,14 @@
# check letsencrypt in docker nginx
docker exec -it central-nginx-1 /bin/bash
ls -l /etc/letsencrypt/archive/odk2.cen-isere.fr/
sudo ls -la /etc/letsencrypt/live/odk2.cen-isere.fr/
# Transfert Certificats Certbot
WEB_ADRESSE="odk2.cen-isere.fr"
ID_CONTAINER=$(docker container ls --all | grep central-nginx-1 | awk '{print $1}')
NUM_PEM=$(sudo ls -Art /etc/letsencrypt/archive/$WEB_ADRESSE | tail -n 1 | grep -o -E '[0-9]+')
sudo docker cp /etc/letsencrypt/archive/$WEB_ADRESSE $ID_CONTAINER:/etc/letsencrypt/archive/
docker exec -it central-nginx-1 sh -c "ln -fs /etc/letsencrypt/archive/$WEB_ADRESSE/privkey$NUM_PEM.pem /etc/letsencrypt/live/$WEB_ADRESSE/privkey.pem"
docker exec -it central-nginx-1 sh -c "ln -fs /etc/letsencrypt/archive/$WEB_ADRESSE/chain$NUM_PEM.pem /etc/letsencrypt/live/$WEB_ADRESSE/chain.pem"
docker exec -it central-nginx-1 sh -c "ln -fs /etc/letsencrypt/archive/$WEB_ADRESSE/fullchain$NUM_PEM.pem /etc/letsencrypt/live/$WEB_ADRESSE/fullchain.pem"
docker exec -it central-nginx-1 sh -c "ln -fs /etc/letsencrypt/archive/$WEB_ADRESSE/cert$NUM_PEM.pem /etc/letsencrypt/live/$WEB_ADRESSE/cert.pem"
13_BDCEN/clean_topographie_site.py
+145
View File
@@ -0,0 +1,145 @@
import geopandas as gpd
from pycen import con_bdcen
###############
#### TOPO #####
###############
sql = '''
ALTER TABLE sites_topo.topo_ligne RENAME column "Geometry" to geom;
ALTER TABLE sites_topo.topo_ligne OWNER TO cen_admin;
CREATE SEQUENCE sites_topo.topo_ligne_id_seq OWNED BY sites_topo.topo_ligne."PK_UID";
SELECT setval('sites_topo.topo_ligne_id_seq', coalesce(max("PK_UID"), 0) + 1, false) FROM sites_topo.topo_ligne;
ALTER TABLE sites_topo.topo_ligne ALTER COLUMN "PK_UID" SET DEFAULT nextval('sites_topo.topo_ligne_id_seq');
'''
with con_bdcen.begin() as cnx:
cnx.execute(sql)
tline_clt = gpd.read_file('/home/colas/Documents/5_BDD/7_CEN38/topographie_consultation.sqlite',layer='topo_ligne')
tline_sai = gpd.read_file('/home/colas/Documents/5_BDD/7_CEN38/topographie_saisie.sqlite',layer='topo_ligne')
tmp = gpd.pd.concat([tline_clt,tline_sai])
isdupl = tmp.geometry.to_wkt().duplicated(keep=False)
tmp[isdupl]
iners = tline_sai.buffer(-1).intersects(tline_clt.unary_union)
(tline_sai[~iners]
.rename_geometry('geom')
.to_postgis(
'topo_ligne',
con_bdcen,
schema='sites_topo',
if_exists='append',
index=False,
))
tpoly_clt = gpd.read_file('/home/colas/Documents/5_BDD/7_CEN38/topographie_consultation.sqlite',layer='topo_polygone')
tpoly_sai = gpd.read_file('/home/colas/Documents/5_BDD/7_CEN38/topographie_saisie.sqlite',layer='topo_polygone')
gpd.pd.concat([tpoly_clt,tpoly_sai]).drop_duplicates(subset='geometry')
iners = tpoly_sai.intersects(tpoly_clt.unary_union)
(tpoly_sai[~iners]
.rename_geometry('geom')
.to_postgis(
'topo_polygone',
con_bdcen,
schema='sites_topo',
if_exists='append',
index=False,
))
tpnt_clt = gpd.read_file('/home/colas/Documents/5_BDD/7_CEN38/topographie_consultation.sqlite',layer='topo_point')
tpnt_sai = gpd.read_file('/home/colas/Documents/5_BDD/7_CEN38/topographie_saisie.sqlite',layer='topo_point')
tmp = gpd.pd.concat([tpnt_clt,tpnt_sai])
isdupl = tmp.geometry.to_wkt().duplicated(keep=False)
tmp[isdupl].drop_duplicates(subset='geometry')
gpd.pd.concat([tpnt_clt,tpnt_sai]).drop_duplicates(subset='geometry')
iners = tpnt_sai.intersects(tpnt_clt.unary_union)
(tpnt_sai[~iners]
.rename_geometry('geom')
.to_postgis(
'topo_point',
con_bdcen,
schema='sites_topo',
if_exists='append',
index=False,
))
###############
#### HYDRO ####
###############
sql = '''
--ALTER TABLE sites_hydro.hydro_point RENAME column "Geometry" to geom;
ALTER TABLE sites_hydro.hydro_point OWNER TO cen_admin;
CREATE SEQUENCE sites_hydro.hydro_point_id_seq OWNED BY sites_hydro.hydro_point."PK_UID";
SELECT setval('sites_hydro.hydro_point_id_seq', coalesce(max("PK_UID"), 0) + 1, false) FROM sites_hydro.hydro_point;
ALTER TABLE sites_hydro.hydro_point ALTER COLUMN "PK_UID" SET DEFAULT nextval('sites_hydro.hydro_point_id_seq');
'''
with con_bdcen.begin() as cnx:
cnx.execute(sql)
hline_clt = gpd.read_file('/home/colas/Documents/5_BDD/7_CEN38/topographie_consultation.sqlite',layer='hydro_ligne')
hline_sai = gpd.read_file('/home/colas/Documents/5_BDD/7_CEN38/topographie_saisie.sqlite',layer='hydro_ligne')
hline_sai.dropna(subset='geometry',inplace=True)
tmp = gpd.pd.concat([hline_clt,hline_sai])
isdupl = tmp.geometry.to_wkt().duplicated(keep=False)
tmp[isdupl]
hline_clt[hline_clt.geometry.to_wkt().duplicated(keep=False)]
hline_sai[hline_sai.geometry.to_wkt().duplicated(keep=False)]
iners = hline_sai.buffer(-1).intersects(hline_clt.unary_union)
(hline_sai[~iners]
.rename_geometry('geom')
.to_postgis(
'hydro_ligne',
con_bdcen,
schema='sites_hydro',
if_exists='append',
index=False,
))
res = gpd.read_postgis('SELECT * FROM sites_hydro.hydro_ligne',con_bdcen)
res['geom_wkt'] = res.geom.to_wkt()
res[res.geom_wkt.duplicated(keep=False)].sort_values(['geom_wkt','Type']).PK_UID.to_list()
hpoly_clt = gpd.read_file('/home/colas/Documents/5_BDD/7_CEN38/topographie_consultation.sqlite',layer='hydro_polygone')
hpoly_sai = gpd.read_file('/home/colas/Documents/5_BDD/7_CEN38/topographie_saisie.sqlite',layer='hydro_polygone')
tmp = gpd.pd.concat([hpoly_clt,hpoly_sai])
isdupl = tmp.geometry.to_wkt().duplicated(keep=False)
tmp[isdupl]
iners = hpoly_sai.buffer(-1).intersects(hpoly_clt.unary_union)
(hpoly_sai
.rename_geometry('geom')
.to_postgis(
'hydro_polygone',
con_bdcen,
schema='sites_hydro',
if_exists='append',
index=False,
))
res = gpd.read_postgis('SELECT * FROM sites_hydro.hydro_polygone',con_bdcen)
res['geom_wkt'] = res.geom.to_wkt()
res[res.geom_wkt.duplicated(keep=False)].sort_values(['geom_wkt','Type','NOM']).PK_UID.to_list()
hpnt_clt = gpd.read_file('/home/colas/Documents/5_BDD/7_CEN38/topographie_consultation.sqlite',layer='hydro_point')
hpnt_clt.dropna(subset='geometry', inplace=True)
hpnt_sai = gpd.read_file('/home/colas/Documents/5_BDD/7_CEN38/topographie_saisie.sqlite',layer='hydro_point')
hpnt_sai.replace({'':None}, inplace=True)
tmp = gpd.pd.concat([hpnt_clt,hpnt_sai])
isdupl = tmp.geometry.to_wkt().duplicated(keep=False)
tmp[isdupl].drop_duplicates(subset='geometry')
gpd.pd.concat([hpnt_clt,hpnt_sai]).drop_duplicates(subset='geometry')
iners = hpnt_sai.intersects(hpnt_clt.unary_union)
(hpnt_sai
.rename_geometry('geom')
.to_postgis(
'hydro_point',
con_bdcen,
schema='sites_hydro',
if_exists='append',
index=False,
))
res = gpd.read_postgis('SELECT * FROM sites_hydro.hydro_point',con_bdcen)
res['geom_wkt'] = res.geom.to_wkt()
res[res.geom_wkt.duplicated(keep=False)].sort_values(['geom_wkt','Type','TYPE_PRINC']).PK_UID.to_list()
14_ANIMATION/r_ter_com.py
+14
View File
@@ -0,0 +1,14 @@
from pycen import con_anm
import geopandas as gpd
from datetime import datetime as dt
com = gpd.read_postgis('SELECT * FROM territoires.vm_communes',con_anm)
ter = gpd.read_postgis('SELECT * FROM territoires.vm_territoires',con_anm)
ter_com = gpd.sjoin(com,ter,how='inner')[['code_insee','territoire_id']]
ter_com['actif'] = True
ter_com['date_maj'] = dt.now().date()
ter_com['utilisateur_id'] = 'cgeier'
ter_com.drop_duplicates(inplace=True)
ter_com.to_sql('r_ter_com',con_anm,schema='territoires',index=False,if_exists='append')
15_POLICE/v_infraction_rnn.py
+85
View File
@@ -0,0 +1,85 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from pycen import con_odk
sql = '''
DROP VIEW IF EXISTS police.v_infraction_rnn;
CREATE VIEW police.v_infraction_rnn AS
WITH photos AS (
SELECT infraction_rnn.json_data ->> '__Submissions-id'::text AS "__Submissions-id",
json_agg((('https://odk2.cen-isere.fr/v1/projects/6/forms/infraction_rnn/submissions/'::text || (infraction_rnn.json_data ->> '__Submissions-id'::text)) || '/attachments/'::text) || (infraction_rnn.json_data ->> 'photos'::text)) AS photos
FROM (
select row_to_json(t.*) json_data
from police.infraction_rnn_photos_repeat_data t
) infraction_rnn
GROUP BY (infraction_rnn.json_data ->> '__Submissions-id'::text)
), strip_infra AS (
SELECT inf_1.json_data ->> '__Submissions-id'::text AS "__Submissions-id",
inf_1.json_data ->> '__id'::text AS __id,
json_strip_nulls(inf_1.json_data) AS json_data
FROM (
select row_to_json(t.*) json_data
from police.infraction_rnn_multiple_infract_data t
) inf_1
), key_infra AS (
SELECT strip_infra.json_data ->> '__Submissions-id'::text AS "__Submissions-id",
strip_infra.json_data ->> '__id'::text AS __id,
json_object_keys(strip_infra.json_data) AS data_keys
FROM strip_infra
), key_infra_agg AS (
SELECT key_infra."__Submissions-id",
key_infra.__id,
array_agg(key_infra.data_keys) AS data_keys
FROM key_infra
GROUP BY key_infra.__id, key_infra."__Submissions-id"
)
SELECT inf.json_data ->> '__id'::text AS id_infraction,
split_part(sub.data_id, ':'::text, 2)::uuid AS id_submission,
sub."submitterName" AS digitiser,
sub.catego_controle,
sub.catego_sensibil,
sub.autre_catego_sensi,
sub.autre_type_sanct,
inf.json_data ->> 'catego_infract'::text AS catego_infract,
inf.json_data ->> key_infra_agg.data_keys[1] AS nature_infra,
CASE
WHEN split_part(key_infra_agg.data_keys[3], '_'::text, 1) = 'natinf'::text
THEN inf.json_data ->> key_infra_agg.data_keys[2]
ELSE NULL::text
END AS code_catego,
CASE
WHEN split_part(key_infra_agg.data_keys[3], '_'::text, 1) = 'natinf'::text
THEN inf.json_data ->> key_infra_agg.data_keys[3]
ELSE inf.json_data ->> key_infra_agg.data_keys[2]
END AS code_infra,
inf.json_data ->> 'descrpt_infract'::text AS descrpt_infract,
sub."Commentaire_remarque_nombre_de_tentes",
sub."Conformit_du_contr_le",
sub.day AS date,
sub.heure,
sub.infract_nb,
sub.nat_operation,
sub.nature_pers,
sub.nb_pers_control,
sub.nb_pers_sensibil,
sub.nb_verb,
sub.nom_agent,
sub.type_sanct,
ph.photos,
sub."submissionDate" AS date_saisie,
sub."updatedAt" AS date_maj,
st_force2d(st_geomfromgeojson(replace(COALESCE(sub.geom, sub.point_carte), '\\'::text, ''::text)))::geometry(Geometry,4326) AS geom
FROM police.infraction_rnn_submissions_data sub
JOIN strip_infra inf ON sub.data_id = inf."__Submissions-id"
JOIN key_infra_agg ON inf.__id = key_infra_agg.__id
JOIN photos ph ON sub.data_id = ph."__Submissions-id"
ORDER BY (split_part(sub.data_id, ':'::text, 2)::uuid), (inf.json_data ->> '__id'::text)
;
GRANT SELECT ON TABLE police.v_infraction_rnn TO grp_police;
'''
with con_odk.begin() as cnx:
cnx.execute(sql)
1_SICEN/MIGRATION GEONATURE/classification_group.py
+322
View File
@@ -0,0 +1,322 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from pycen import con_sicen
import geopandas as gpd
DICT_TAXREF_OLDNAME = {
116744 : 521658, # Quercus petraea
# 9999056 : 0, # Micromammalia sp.
9999077 : 183820, # Arachnida sp.
9999078 : 187496, # Fungi sp.
# 9999081 : 0, # Heterocera sp.
9999058 : 195005, # Myotis sp.
}
DICT_TAXREF = {
# 116744:{'cd_nom':521658,'lb_nom':'Quercus petraea (Matt.) Liebl., 1784'}, # Quercus petraea
# 9999077:{'cd_nom':183820,'lb_nom':'Arachnida Cuvier, 1812'}, # Arachnida sp.
# Espèce domestique
9999031 : {'cd_nom':441709,'lb_nom':'Cairina moschata (Linnaeus, 1758)'}, # Cairina moschata f. domestica
# GROUPE
9999005: {'cd_nom':194069,'lb_nom':'Leucanthemum Mill., 1754'}, # Leucanthemum vulgare (#groupe) (#ref)
9999014: {'cd_nom':197281,'lb_nom':'Rubus L., 1753 [nom. et typ. cons.]'}, # Rubus fruticosus (#groupe) (#ref)
9999017: {'cd_nom':198226,'lb_nom':'Taraxacum F.H.Wigg., 1780'}, # Taraxacum officinale (#groupe) (#ref)
9999019: {'cd_nom':198449,'lb_nom':'Thymus L., 1753'}, # Thymus serpyllum (#groupe) (#ref)
9999020: {'cd_nom':198902,'lb_nom':'Vicia L., 1753'}, # Vicia sativa (#groupe) (#ref)
# COMPLEXE
9999075: {'cd_nom':196296,'lb_nom':'Pipistrellus Kaup, 1829'}, # Pipistrellus nathusii / kuhlii (#complexe) (#ref)
9999074: {'cd_nom':197040,'lb_nom':'Rana Linnaeus, 1758'}, # Rana dalmatina / temporaria (#complexe) (#ref)
9999057: {'cd_nom':195005,'lb_nom':'Myotis Kaup, 1829'}, # Myotis myotis / blythii (#complexe) (#ref)
9999050: {'cd_nom':193993,'lb_nom':'Leptidea Billberg, 1820'}, # Leptidea sinapis / reali (#complexe) (#ref)
9999066: {'cd_nom':196980,'lb_nom':'Pyrgus Hübner, 1819'}, # Pyrgus malvae / malvoides (#complexe) (#ref)
9999063: {'cd_nom':444436,'lb_nom':'Pelophylax Fitzinger, 1843'}, # Pelophylax kl. esculentus / lessonae (#complex...
9999054: {'cd_nom':194481,'lb_nom':'Martes Pinel, 1792'}, # Martes martes / foina (#complexe) (#ref)
9999037: {'cd_nom':191029,'lb_nom':'Colias Fabricius, 1807'}, # Colias alfacariensis / hyale (#complexe) (#ref)
9999023: {'cd_nom':188772,'lb_nom':'Acrocephalus J. A. Naumann & J. F. Naumann, 1811'}, # Acrocephalus palustris / scirpaceus (#complexe...
9999064: {'cd_nom':4280,'lb_nom':'Phylloscopus collybita (Vieillot, 1817)'}, # Phylloscopus collybita tristis / "abietinus" ...
9999046: {'cd_nom':192539,'lb_nom':'Felis Linnaeus, 1758'}, # Felis silvestris / catus (#complexe) (#ref)
9999080: {'cd_nom':194357,'lb_nom':'Lysandra Hemming, 1933'},# Lysandra coridon / hispana (#complexe)
9999033: {'cd_nom':886228,'lb_nom':'Acanthis Borkhausen, 1797'}, # Carduelis flammea flammea / cabaret / Carduelis hornemanni (#complexe)
9999041: {'cd_nom':4503,'lb_nom':'Corvus corone Linnaeus, 1758'}, # Corvus corone corone / cornix (#complexe)
9999042: {'cd_nom':186239,'lb_nom':'Vespertilionidae J. E. Gray, 1821'}, # Eptesicus / Nyctalus sp. (#complexe) (#ref)
9999082: {'cd_nom':195005,'lb_nom':'Myotis Kaup, 1829'}, # Myotis daubentonii/Myotis mystacinus (#complexe)
9999083: {'cd_nom':699094,'lb_nom':'Yangochiroptera Koopman, 1985'}, # Pipistrellus/Miniopterus (#complexe)
}
def get_pheno_genre(esp_fx, df):
nom = esp_fx
return
def _vm_synthese_observations_(where=''):
# Lecture des données
sql = 'SELECT * FROM saisie.vm_synthese_observations '
df = gpd.read_postgis(sql+where,con_sicen)
# drop Micromammalia sp.
drp = df.loc[df.cd_nom == 9999056].index
df.drop(drp,inplace=True)
is_group = df.nom_complet.str.contains('#group')
is_complex = df.nom_complet.str.contains('#complex')
is_domestic = df.nom_complet.str.contains('domestic')
lst_group = [*df[is_group].nom_complet.unique()]
lst_complex = [*df[is_complex].nom_complet.unique()]
lst_domestic = [*df[is_domestic].nom_complet.unique()]
# df['group'] = None
# df['complex'] = None
# df.loc[is_group,'group'] = df[is_group].nom_complet.copy()
# df.loc[is_complex,'complex'] = df[is_complex].nom_complet.copy()
df['rmq_taxonomie'] = None
df.loc[is_complex|is_group|is_domestic,'rmq_taxonomie'] = df[is_complex|is_group|is_domestic].nom_complet.copy()
for cd in df[is_complex|is_group|is_domestic].cd_nom.unique():
lb = df[df.cd_nom==cd].nom_complet.unique()[0]
cdn = {cd:DICT_TAXREF[cd]['cd_nom']}
lbn = {lb:DICT_TAXREF[cd]['lb_nom']}
df.nom_complet.replace(lbn,inplace=True)
df.nom_latin.replace(lbn,inplace=True)
df.cd_nom.replace(cdn,inplace=True)
# Transform geom TO WKT format
df['geom_wkt'] = df.geometry.to_wkt()
# Récup code insee département
df['insee_dep'] = df.insee_commune.str[:2]
# Mise en forme de la précision de la données
df.precision = format_precision(df.precision)
# Actualisation des cd_nom
df.cd_nom.replace(DICT_TAXREF_OLDNAME,inplace=True)
# Mise en forme des effectifs
df.effectif = format_effectif(df.effectif.copy())
df.effectif_min = format_effectif(df.effectif_min.copy())
df.effectif_max = format_effectif(df.effectif_max.copy())
df.loc[df.effectif.isna(),'effectif'] = df[df.effectif.isna()].effectif_min
df.effectif.fillna('1',inplace=True)
df.loc[df.effectif_max.isna(),'effectif_max'] = df[df.effectif_max.isna()].effectif
df.effectif = df.effectif.astype(int)
df.effectif_max = df.effectif_max.astype(int)
check_effectif = df['effectif'].astype(float) > df['effectif_min'].astype(float)
if not df[check_effectif].empty:
print('WARNING : "effectif" > "effectif_min"')
print(' nrows : %s'%df[check_effectif].shape[0])
df.loc[check_effectif,'effectif_max'] = df[check_effectif].effectif.copy()
df.loc[check_effectif,'effectif'] = df[check_effectif].effectif_min.copy()
print( 'effectif TO effectif_max : OK !')
df.effectif_min = df.effectif.copy()
return df
def format_precision(lst):
return lst.copy().replace({
'GPS': '5',
'0 à 10m': '5',
'10 à 100m': '50',
'100 à 500m': '250',
'500 à 1000m': '750',
'> 1000m': '2000',
'lieu-dit': '2500',
'commune': '5000',
})
def format_effectif(lst):
lst.loc[lst.notna()] = lst[lst.notna()].astype(int).astype(str)
return lst
def additional_data(df,columns):
df['additional_data'] = df[columns].to_json(orient="records",force_ascii=False)
df.drop(columns=columns,inplace=True)
return df
class flore:
def all():
w = "WHERE regne = 'Plantae'"
res = _vm_synthese_observations_(w)
# return additional_data(res,['strate_flore','effectif_textuel'])
return res
def vasculaire():
w = "WHERE group1_inpn = 'Plantes vasculaires'"
res = _vm_synthese_observations_(w)
# return additional_data(res,['strate_flore','effectif_textuel'])
return res
def bryophyte():
w = "WHERE group1_inpn = 'Bryophytes'"
res = _vm_synthese_observations_(w)
# return additional_data(res,['strate_flore','effectif_textuel'])
return res
def characee():
w = "WHERE famille = 'Characeae'"
res = _vm_synthese_observations_(w)
# return additional_data(res,['strate_flore','effectif_textuel'])
return res
class faune:
def all():
w = "WHERE regne = 'Animalia'"
return _vm_synthese_observations_(w)
class invertebre:
def all():
w = "WHERE regne = 'Animalia' AND phylum <> 'Chordata'"
return _vm_synthese_observations_(w)
def odonate():
w = "WHERE ordre = 'Odonata'"
return _vm_synthese_observations_(w)
def orthoptere():
w = "WHERE ordre = 'Orthoptera'"
return _vm_synthese_observations_(w)
def lepidoptere():
w = "WHERE ordre = 'Lepidoptera'"
return _vm_synthese_observations_(w)
def mollusque():
w = "WHERE group1_inpn = 'Mollusques'"
return _vm_synthese_observations_(w)
def crustacees():
w = "WHERE group2_inpn = 'Crustacés'"
return _vm_synthese_observations_(w)
def ascomycete():
w = "WHERE ordre = 'Ascomycètes'"
return _vm_synthese_observations_(w)
def autre():
w = """
WHERE regne = 'Animalia'
AND phylum <> 'Chordata'
AND ordre <> 'Odonata'
AND ordre <> 'Orthoptera'
AND ordre <> 'Lepidoptera'
AND group1_inpn <> 'Mollusques'
AND group2_inpn <> 'Crustacés'
AND ordre <> 'Ascomycètes'
"""
return _vm_synthese_observations_(w)
class vertebre:
def all():
w = "WHERE regne = 'Animalia' AND phylum = 'Chordata'"
return _vm_synthese_observations_(w)
def mammifere():
w = "WHERE group2_inpn = 'Mammifères' AND ordre <> 'Chiroptera'"
return _vm_synthese_observations_(w)
def chiroptere():
w = "WHERE ordre = 'Chiroptera'"
return _vm_synthese_observations_(w)
def oiseau():
w = "WHERE group2_inpn = 'Oiseaux'"
return _vm_synthese_observations_(w)
def reptile():
w = "WHERE group2_inpn = 'Reptiles'"
return _vm_synthese_observations_(w)
def amphibien():
w = "WHERE group2_inpn = 'Amphibiens'"
return _vm_synthese_observations_(w)
def poisson():
w = "WHERE group2_inpn = 'Poissons'"
return _vm_synthese_observations_(w)
def champignon():
w = "WHERE regne IN ('Fungi','Chromista')"
return _vm_synthese_observations_(w)
def export_csv(df,path_name):
df.dropna(how='all',axis=1,inplace=True)
drop_cols = [
'regne','phylum','classe','ordre','famille','group1_inpn','group2_inpn','cd_ref',
'altitude_z','longitude_x','latitude_y','geom','nom_commune',
'etude','lot_donnee','observateurs','structures'
]
d = df.columns[df.columns.isin(drop_cols)]
df.drop(columns=d,inplace=True)
if df.shape[0] > 50000:
from math import ceil
loop = ceil(df.shape[0]/50000)
i = 0
for j in range(1,loop+1):
p = path_name.replace('.csv','_%s.csv'%j)
jj = j*50000
df.iloc[i:jj].to_csv(p,index=False)
i = jj
else:
df.to_csv(path_name,index=False)
if __name__ == "__main__":
test = False
PATH = '/home/colas/Documents/9_PROJETS/4_SICEN/GN_MIGRATION/'
al = flore.all()
va = flore.vasculaire()
br = flore.bryophyte()
ch = flore.characee()
export_csv(al,PATH+'FLORE/all.csv')
# export_csv(va[va.effectif< va.effectif_max].copy(),PATH+'FLORE/test_eff_vasculaire.csv')
export_csv(va,PATH+'FLORE/vasculaire.csv')
export_csv(br,PATH+'FLORE/bryophyte.csv')
export_csv(ch,PATH+'FLORE/characee.csv')
al.shape[0] - (va.shape[0]+br.shape[0]+ch.shape[0])
fa = faune.all()
fia = faune.invertebre.all()
fva = faune.vertebre.all()
fa.shape[0] - (fia.shape[0] + fva.shape[0])
if test :
fa[~fa.id_obs.isin([*fia.id_obs,*fva.id_obs])]
ca = champignon()
vm = _vm_synthese_observations_()
date_cols = vm.columns[vm.columns.str.contains('date')]
mtd = vm[['lot_donnee','protocole','etude']].drop_duplicates()
vm.shape[0] - (al.shape[0]+fa.shape[0]+ca.shape[0])
if test :
vm[~vm.id_obs.isin([*al.id_obs,*fa.id_obs,*ca.id_obs])]
# Sub verification
fiod = faune.invertebre.odonate()
fior = faune.invertebre.orthoptere()
file = faune.invertebre.lepidoptere()
fimo = faune.invertebre.mollusque()
ficr = faune.invertebre.crustacees()
fias = faune.invertebre.ascomycete()
fiau = faune.invertebre.autre()
fia.shape[0] - (
fiod.shape[0] + fior.shape[0] + file.shape[0] + fimo.shape[0] + ficr.shape[0] + fias.shape[0] + fiau.shape[0]
)
if test :
fia[~fia.id_obs.isin([*fiod.id_obs,*fior.id_obs,*file.id_obs,*fimo.id_obs,*ficr.id_obs,*fias.id_obs,*fiau.id_obs])]
fvma = faune.vertebre.mammifere()
fvch = faune.vertebre.chiroptere()
fvoi = faune.vertebre.oiseau()
fvre = faune.vertebre.reptile()
fvam = faune.vertebre.amphibien()
fvpo = faune.vertebre.poisson()
fva.shape[0] - (
fvma.shape[0] + fvch.shape[0] + fvoi.shape[0] + fvre.shape[0] + fvam.shape[0] + fvpo.shape[0]
)
if test :
fva[~fva.id_obs.isin([*fvma.id_obs,*fvch.id_obs,*fvoi.id_obs,*fvre.id_obs,*fvam.id_obs,*fvpo.id_obs])]
1_SICEN/MIGRATION GEONATURE/sicen_to_gn.py
+860
View File
@@ -0,0 +1,860 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from pycen import update_to_sql, con_sicen,con_gn
import geopandas as gpd
import os
DIR = '/home/colas/Documents/9_PROJETS/4_SICEN/GN_MIGRATION'
def form_cdnom(data):
dict_cdnom = {
116744:521658, # Flore : Quercus petraea
}
data.replace({'cd_nom':dict_cdnom},inplace=True)
data.replace({'cd_ref':dict_cdnom},inplace=True)
def form_complx_grp(data):
dict_cdnom = {
9999005:105817, # Leucanthemum vulgare (#groupe)
9999014:119097, # Rubus fruticosus (#groupe)
9999017:717630, # Taraxacum officinale (#groupe)
9999019:126573, # Thymus serpyllum (#groupe)
9999020:129298, # Vicia sativa (#groupe)
9999023:188772, # Acrocephalus palustris / scirpaceus (#complexe)
9999031:441709, # Cairina moschata f. domestica (#forme)
9999033:190350, # Carduelis flammea flammea / cabaret / Carduelis hornemanni (#complexe)
9999037:191029, # Colias alfacariensis / hyale (#complexe)
9999041:4503, # Corvus corone corone / cornix (#complexe)
9999042:186239, # Eptesicus / Nyctalus sp. (#complexe)
9999046:192539, # Felis silvestris / catus (#complexe)
9999050:193993, # Leptidea sinapis / reali (#complexe)
9999054:194481, # Martes martes / foina (#complexe)
9999057:195005, # Myotis myotis / blythii (#complexe)
9999063:444436, # Pelophylax kl. esculentus / lessonae (#complexe)
9999064:4280, # Phylloscopus collybita tristis / "abietinus" (#complexe)
9999066:196980, # Pyrgus malvae / malvoides (#complexe)
9999074:197040, # Rana dalmatina / temporaria (#complexe)
9999075:196296, # Pipistrellus nathusii / kuhlii (#complexe)
9999080:194357, # Lysandra coridon / hispana (#complexe)
9999082:195005, # Myotis daubentonii / Myotis mystacinus (#complexe)
9999083:699094, # Pipistrellus / Miniopterus (#complexe)
}
lst_cdnom_old = [*dict_cdnom.keys()]
if data.cd_nom.isin(lst_cdnom_old).any():
data.loc[data.cd_nom.isin(lst_cdnom_old),'complexe_groupe'] = data[data.cd_nom.isin(lst_cdnom_old)].nom_complet
data.replace({'cd_nom':dict_cdnom},inplace=True)
sql = 'SELECT cd_nom, nom_complet nom_new FROM taxonomie.taxref where cd_nom in {}'.format(tuple(dict_cdnom.values()))
tax = gpd.pd.read_sql(sql, con_gn)
data = data.merge(tax,how='left',on='cd_nom')
data.loc[data.nom_new.notna(),'nom_latin'] = data[data.nom_new.notna()].nom_new
data.drop(columns='nom_new',inplace=True)
def correct_taxonomie(data):
dict_taxo = {
444440:'Pelophylax kl\. esculentus \(Linnaeus, 1758\)'
}
for k,v in dict_taxo.items():
is_saisie = data.rmq_observation.str.contains(v,na=False)
if not is_saisie.any():
continue
data.loc[is_saisie,'nom_latin'] = v
data.loc[is_saisie,'cd_nom'] = k
data.loc[is_saisie,'cd_ref'] = k
def form_precision(data):
dict_pre = {
'GPS':0,
'0 à 10m':10,
'10 à 100m':100,
'100 à 500m':500,
'lieu-dit':750,
'commune':None,
}
is_com = data.precision =='commune'
rmq_null = data.rmq_localisation.isnull()
data.loc[is_com&rmq_null,'rmq_localisation'] = 'Localisation : commune'
data.loc[is_com&~rmq_null,'rmq_localisation'] = 'Localisation : commune ;'+data[is_com].rmq_localisation
data.replace({'precision':dict_pre},inplace=True)
data.precision = data.precision.astype('Int64')
def form_effectif(data):
eff_notna = data.effectif.notna()
efmin_notna = data.effectif_min.notna()
efmax_isna = data.effectif_max.isna()
data.loc[(~eff_notna)&efmin_notna,'effectif'] = data[(~eff_notna)&efmin_notna].effectif_min
data.loc[(~eff_notna)&(~efmax_isna),'effectif'] = data[(~eff_notna)&(~efmax_isna)].effectif_max
# data.loc[efmax_isna,'effectif_max'] = data[efmax_isna].effectif
data.effectif = data.effectif.astype('Int64')
data.effectif_max = data.effectif_max.astype('Int64')
t1 = data.effectif_min < data.effectif
data.loc[t1,'effectif_max'] = data[t1].effectif
data.loc[t1,'effectif'] = data[t1].effectif_min
# Est trop incertain... Les effectifs récupérés sont < aux effectifs déclarés
# ==> Incompréhansion, Ne sera pas utilisé
def recup_faune_eff(data):
# Récupération des effectifs standardisés du champ rmq_observation
ismixte = data.rmq_observation.str.contains('\\d.00 Mâle\(s\) et \\d.00 Femelle\(s\)',na=False)
ismal = data.rmq_observation.str.contains('\\d.00 Mâle\(s\) et \? Femelle\(s\)',na=False)
isfem = data.rmq_observation.str.contains('\? Mâle\(s\) et \\d.00 Femelle\(s\)',na=False)
data.loc[ismixte,'extract_eff'] = (data[ismixte].rmq_observation
.str.extract('(\d+).00 Mâle\(s\) et (\d+).00 Femelle\(s\)').astype(int).sum(axis=1))
data.loc[ismal,'extract_eff'] = (data[ismal].rmq_observation
.str.extract('(\d+).00 Mâle\(s\) et \? Femelle\(s\)').astype(int).sum(axis=1))
data.loc[isfem,'extract_eff'] = (data[isfem].rmq_observation
.str.extract('\? Mâle\(s\) et (\d+).00 Femelle\(s\)').astype(int).sum(axis=1))
extr = data.extract_eff.notna()
extr_sup = data.effectif < data.extract_eff
data[extr_sup]
age_na = data.age_faune.isna()
no_0 = data.extract_eff > 0
if any(age_na&no_0&ismixte):
data.loc[age_na&no_0&ismixte,'age_faune'] = 'mixte'
if any(age_na&no_0&ismal):
data.loc[age_na&no_0&ismal,'age_faune'] = 'male'
if any(age_na&no_0&isfem):
data.loc[age_na&no_0&isfem,'age_faune'] = 'femelle'
def form_date(data):
cols = data.columns[data.columns.str.contains('date')]
for col in cols:
data.loc[data[col].isna(),col] = None
def recup_stadevie(data):
dict_repro = {
'ODO_Exuvie/émergence':'Exuvie/émergence',
'ODO_Immature':'Immature',
**dict.fromkeys(['ODO_Mâles+Femelles','ODO_Tandem','ODO_Territorial','ODO_Ponte'],'Adulte'),
**dict.fromkeys(['CHIR_Indéterminé','ODO_Indéterminé'],'Indéterminé')
}
if 'age_faune' in data.columns:
age_isna = data.age_faune.isna()
age_inrmq = data.rmq_observation.str.contains('Stade de vie')
data.loc[age_isna&age_inrmq,'age_faune'] = (data[age_isna&age_inrmq].rmq_observation
.str.split('Stade de vie :')
.str[1]
.str.split('|')
.str[0]
.str.strip())
double_info = data.age_faune.isin(['Imago, adulte', 'Nymphe, immature'])
d2 = data[double_info].copy()
data.age_faune = data.age_faune.str.replace('Imago, adulte','Imago')
data.age_faune = data.age_faune.str.replace('Nymphe, immature','Nymphe')
d2.age_faune = d2.age_faune.str.replace('Imago, adulte','Adulte')
d2.age_faune = d2.age_faune.str.replace('Nymphe, immature','Immature')
d2.id_obs = ('9999'+d2.id_obs.astype(str)).astype(int)
data = gpd.pd.concat([data,d2])
lst_age = [*dict_repro.keys()]
age_isna = data.age_faune.isna()
t1 = age_isna&(data.reprostatut_faune.isin(lst_age))
data.loc[t1,'age_faune'] = data[t1].reprostatut_faune.replace(dict_repro)
t2 = data.age_faune=='Exuvie/émergence'
is_exuvie = (data.rmq_observation
.replace(['é','E','É'],'e',regex=True)
.str.contains('exuvie',na=False))
is_emerge = (data.rmq_observation
.replace(['é','E','É'],'e',regex=True)
.str.contains('emerge',na=False))
rmq_isna = data.rmq_observation.isna()
data.loc[is_exuvie&~is_emerge&t2,'age_faune'] = 'Exuvie'
data.loc[~is_exuvie&is_emerge&t2,'age_faune'] = 'Emergent'
data.loc[is_exuvie&is_emerge&t2,'age_faune'] = 'Exuvie'
data.loc[is_exuvie&is_emerge&rmq_isna&t2,'age_faune'] = 'Emergent'
t3 = data.age_faune=='Oeuf/ponte/larve/nymphe/chenille...'
# Odonate
is_odo = data.ordre == 'Odonata'
data.loc[t3&is_odo,'age_faune'] = 'Exuvie'
# Amphibiens
is_amphi = data.group2_inpn == 'Amphibiens'
is_larve = data.rmq_observation.str.contains('larve',na=False)
is_ponte = data.rmq_observation.str.contains('ponte|Ponte',na=False)
is_tetar = data.rmq_observation.replace(['ê','é'],'e',regex=True).str.contains('tetard',na=False)
is_urode = data.ordre=='Urodela'
is_anure = data.ordre=='Anura'
rmq_na = data.rmq_observation.isna()
data.loc[t3&is_amphi&is_larve,'age_faune'] = 'Larve'
data.loc[t3&is_amphi&is_ponte,'age_faune'] = 'Oeufs'
data.loc[t3&is_amphi&is_tetar,'age_faune'] = 'Têtard'
data.loc[t3&is_amphi&is_urode&rmq_na,'age_faune'] = 'Larve'
data.loc[t3&is_amphi&is_anure&rmq_na,'age_faune'] = 'Ponte'
# Corresction de 3 données de Nicolas Biron
t3 = data.age_faune=='Oeuf/ponte/larve/nymphe/chenille...'
data.loc[t3&(is_urode|is_anure),'age_faune'] = 'Ponte'
# Correction de la dernière donnée restante (Marjorie Simean)
t3 = data.age_faune=='Oeuf/ponte/larve/nymphe/chenille...'
data.loc[t3,'age_faune'] = 'Oeufs'
# Définition de l'objet du dénombrement si ponte
if 'obj_denombre' not in data.columns:
data['obj_denombre'] = None
isnot_ponte = ~data.rmq_observation.str.contains('nbre pontes:0',na=False)
data.loc[is_amphi&is_ponte&isnot_ponte,'obj_denombre'] = 'Ponte'
# Récup stade_vie derrirère le terme @NSA
age_isna = data.age_faune.isna()
as_nsa = data.rmq_observation.str.contains('@NSA',na=False)
# data.loc[age_isna&as_nsa,'test'] = (data[age_isna&as_nsa].rmq_observation
# .str.split('@NSA :|@NSA:')
# .str[1]
# .str.split('/|;')
# .str[0]
# .str.strip())
data.loc[age_isna&as_nsa,'age_faune'] = (data[age_isna&as_nsa].rmq_observation
.str.split('@NSA :|@NSA:')
.str[1]
.str.split('/|;')
.str[0]
.str.strip()
.str.lower()
.str.extract('(imm|juv|ad|jeune|larve)')[0])
# Récup stade_vie derrirère le terme Commentaires
age_isna = data.age_faune.isna()
as_com = data.rmq_observation.str.contains('Commentaires : \\d',na=False)
data.loc[age_isna&as_com,'age_faune'] = (data[age_isna&as_com].rmq_observation
.str.split('[,(.]|Code Atlas')
.str[0].str.strip()
.str.lower()
.replace(
{'femelles':'femelle','mâles':'mâle','adultes':'adulte',
'jeunes':'jeune','juvéniles':'juvénile','larves':'larve',
'couples':'couple','x':''},regex=True)
.str.extract('[Commentaires : \d+ ](jeune volant|juvénile|juv|couple|adulte|ad|crapelet|femelle adulte|larve|mâle juvénile|mâle adulte|subadulte|larve \/ têtard)$')[0])
age_isna = data.age_faune.isna()
data.loc[age_isna&as_com,'age_faune'] = (data[age_isna&as_com].rmq_observation
.str.split('[,(.]|Code Atlas')
.str[0].str.strip()
.str.lower()
.replace(
{'femelles':'femelle','mâles':'mâle','adultes':'adulte',
'jeunes':'jeune','juvéniles':'juvénile','larves':'larve',
'têtards':'têtard','couples':'couple','x':''},regex=True)
.str.extract('[Commentaires : \d+](jeune volant|juvénile|juv|couple|adulte|ad|crapelet|femelle adulte|larve|mâle adulte|subadulte|larve / têtard)$')[0])
# Précision de larve / têtard
is_impreci = data.age_faune=='larve / têtard'
data.loc[is_impreci&is_anure,'age_faune'] = 'têtard'
data.loc[is_impreci&is_urode,'age_faune'] = 'larve'
data.age_faune = (data.age_faune
.replace({
'ad':'Adulte',
'Oeufs':'Oeuf',
'imm':'Immature',
'jeune':'Jeune',
'juv':'Juvénile',
'subadulte':'Sub-adulte',
'Subadulte':'Sub-adulte',})
.str.capitalize()
)
sex_isna = data.sexe_faune.isna()
is_male = data.age_faune.str.contains('Mâle',na=False)
is_feml = data.age_faune.str.contains('Femelle',na=False)
is_copl = data.age_faune.str.contains('Couple',na=False)
is_immat = data.age_faune.str.contains('Immature',na=False)
data.loc[sex_isna&is_male,'sexe_faune'] = 'Mâle'
data.loc[sex_isna&is_feml,'sexe_faune'] = 'Femelle'
data.loc[sex_isna&is_copl,'sexe_faune'] = 'Couple'
data.loc[sex_isna&is_immat,'sexe_faune'] = 'Immature'
return data
def complete_objdenomb(data):
asnot_denomb = data.obj_denombre.isna()
isnot_dead = data.etat_bio != 'mort'
as_statbio = data.statut_bio.notna()
as_compt = data.comportement.notna()
as_atlas = data.code_atlas.notna()
as_age = data.age_faune.notna()
as_sexe = data.sexe_faune.notna()
as_repro = data.reprostatut_faune.notna()
as_determ = data.determination.notna()
data.loc[asnot_denomb&isnot_dead&(as_statbio|as_compt|as_atlas|as_age|as_sexe|as_repro|as_determ),'obj_denombre'] = 'individu'
return data
def recup_comptmt(data):
if 'reprostatut_faune' in data.columns:
# Récupération du comportement après `Comp. ind` dans la colonne `rmq_observation`
compt_isna = data.reprostatut_faune.isna()
in_rmq = data.rmq_observation.str.contains('Comp. ind')
data.loc[compt_isna&in_rmq,'comportement'] = (data[compt_isna&in_rmq].rmq_observation
.str.split('Comp. ind. :')
.str[1]
.str.split('|')
.str[0]
.str.strip())
# Récupération du comportement dans la colonne `reprostatut_faune`
sex_isna = data.sexe_faune.isna()
comp_isna = data.comportement.isna()
lst_comp_inv = [
'ODO_Tandem','ODO_Territorial','INV_Accouplement','INV_Chant (orthoptères)','ODO_Ponte']
lst_comp_ver = ['OIS_Reproduction certaine (patrim)','MAM_Fécès - épreinte - urine', 'OIS_Nid avec jeune(s)',
'OIS_Couvaison', 'CHIR_Hivernant', 'AMP_Mise bas - ponte','CHIR_Lethargie', 'OIS_Chant', # 'AMP_Indéterminé',
'AMP_Chant','CHIR_Estivant', 'CHIR_Transit', 'OIS_Reproduction possible',
'MAM_Alarme', 'AMP_Accouplement', 'OIS_Fuite - envol','OIS_Accouplement']
if data.reprostatut_faune.isin(lst_comp_inv).any():
t3 = (~compt_isna)&comp_isna&sex_isna&(data.reprostatut_faune.isin(lst_comp_inv))
elif data.reprostatut_faune.isin(lst_comp_ver).any():
t3 = (~compt_isna)&comp_isna&(data.reprostatut_faune.isin(lst_comp_ver))
data.loc[t3,'comportement'] = data[t3].reprostatut_faune
# Récupération du comportement après `Comp. ind` dans la colonne `rmq_observation`
# dictionnaire des transports
dict_transport = {
**dict.fromkeys(['transport de branche','transport de matériaux','transport de brindille','transporte une branche',"transport d'une branche",'transportant des matériaux','transportant une branche','transporte des feuilles'],'transport matériaux'),
**dict.fromkeys(['transport de nourriture','transport nourriture','transport de poisson','transportant nourriture','Transporte une proie','transport de proie'],'transport nourriture'),
}
lst_rmq_comp = "chant|houspille|en chasse|{}|en vol|en survol|posé|parade|s'envole|se nourrit|se pose|tambourinage|vol migratoire|hivernage|nourrissage|couveur|nourrit|cht|cris|estivant|hivernant|Construction de nid|alarme|migration active".format('|'.join(dict_transport.keys()))
comp_isna = data.comportement.isna()
in_rmq = data.rmq_observation.str.contains('[%s]'%lst_rmq_comp,case=False,na=False)
data.loc[comp_isna&in_rmq,'comportement'] = (data[comp_isna&in_rmq].rmq_observation
.str.lower()
.str.extract('(%s)'%lst_rmq_comp)[0]
.replace(dict_transport))
comp_isna = data.comportement.isna()
envol = data.rmq_observation.str.contains('Statut:Passage.')
if any(comp_isna&envol):
data.loc[comp_isna&envol,'comportement'] = 'en vol'
comp_isna = data.comportement.isna()
data.loc[comp_isna,'comportement'] = (data[comp_isna].rmq_observation
.replace({'x',''},regex=True)
.str.lower()
.str.extract('[détails : \d+ ]\((en transit)\)')[0])
def _get_gn_atlas():
sch = 'ref_nomenclatures'
tab = ['t_nomenclatures','bib_nomenclatures_types']
bib_mnemo='CODE_ATLAS'
sql = """
SELECT
a.id_nomenclature,
a.cd_nomenclature,
a.mnemonique mnemo,
a.label_default as label,
a.label_fr,
a.definition_default definition,
a.definition_fr def_fr,
b.mnemonique bib_mnemo,
b.label_default bib_label,
b.definition_default bib_def,
a.active
FROM {sch}.{tab0} a
JOIN {sch}.{tab1} b USING (id_type)
""".format(sch=sch,tab0=tab[0],tab1=tab[1])
if bib_mnemo is not None:
sql += " WHERE "
sql += "b.mnemonique = '%s'"%bib_mnemo
return gpd.pd.read_sql_query(sql,con_gn)
def recup_atlas(data):
is_atlas = data.rmq_observation.str.contains('Code atlas',na=False)
# is_atlas = data.rmq_observation.str.contains('Code atlas :',na=False)
if any(is_atlas):
atlas = _get_gn_atlas()
dict_code = dict(zip(atlas.cd_nomenclature.astype(int).astype(str),atlas.label_fr))
dict_label = dict(zip(atlas.label.str.removesuffix('.'),atlas.label_fr))
dict_autre = {
'Mâle chanteur (ou cris de nidification) ou tambourinage en période de reproduction':'Mâle chanteur présent en période de nidification, cris nuptiaux ou tambourinage entendus, mâle vu en parade',
'Jeunes fraîchement envolés (espèces nidicoles) ou poussins (espèces nidifuges)':'Jeunes en duvet ou jeunes venant de quitter le nid et incapables de soutenir le vol sur de longues distances'
}
data.loc[is_atlas,'code_atlas'] = (data[is_atlas].rmq_observation
.str.split('Code atlas')
.str[1]
# .str.split('\||@|;|\(')
.str.split(r'[|@;]')
.str[0]
.replace({
**dict.fromkeys([':','Nidification possible \(','Nidification certaine \(','Nidification probable \('],''),
'\s+':' ',
' - ':'; '},regex=True)
.str.strip()
.str.replace('nuptial parades','nuptial: parades')
.str.removesuffix(')')
.replace('',None)
.replace(dict_code)
.replace(dict_autre)
.replace(dict_label))
data.loc[is_atlas,'rmq1'] = (data[is_atlas].rmq_observation
.str.split('Code atlas')
.str[0]
.str.strip()
.replace(dict.fromkeys(['Commentaires :','Remarques :','Détails : @','','nan'],None))
)
data.loc[is_atlas,'rmq2'] = (data[is_atlas].rmq_observation
.str.split('Code atlas')
.str[1]
.str.split(r'[|@;]')
.str[1]
.str.strip()
.str.removesuffix(';')
.str.strip()
.replace(dict.fromkeys(['','nan'],None))
)
data.loc[is_atlas,'rmq_observation'] = [
' ; '.join([x,y]) if not gpd.pd.isna(y) and not gpd.pd.isna(x)
else y if gpd.pd.isna(x)
else x
for x,y in zip(data[is_atlas].rmq1,data[is_atlas].rmq2)
]
data.rmq_observation.replace('nan',None,inplace=True)
data.drop(columns=['rmq1','rmq2'],inplace=True)
atlas_isna = data.code_atlas.isna()
rmq_atlas = {
'2':"Observation de l'espèce pendant la période de nidification dans un biotope adéquat",
'2':"Observation de l'espèce pendant la période de nidification",
'4':'Couple pendant la période de nidification dans un biotope adéquat',
}
for k,v in rmq_atlas.items():
as_atlas = data.rmq_observation.str.contains(v,na=False)
data.loc[atlas_isna&as_atlas,'code_atlas'] = dict_code[k]
# Récupération du code atlas 99
for i in ['99','1.00','2','3.00','3 ','4 ','4.00','5.00','5 ',
'6','7','8','9.00','9 ','10','11 ','12 ','12.00','13',
'14.00','14 ','15.00','16','17','18','19']:
is_atlas = data.rmq_observation.str.contains('Code Atlas : %s'%i,na=False)
atlas_isna = data.code_atlas.isna()
if is_atlas.any():
data.loc[is_atlas&atlas_isna,'code_atlas'] = str(int(float('%s'%i)))
data.loc[is_atlas&atlas_isna,'code_atlas'] = data[is_atlas&atlas_isna].code_atlas.replace(dict_code)
# définition du code_atlas par le statut repro
lst_repro = [
'Reproduction possible','Reproduction probable','Reproduction confirmée','Nicheur possible'
]
atlas_isna = data.code_atlas.isna()
is_aves = data.group2_inpn == 'Oiseaux'
is_repro = data.statut_bio.isin(lst_repro)
if any(is_aves&is_repro&atlas_isna):
data.loc[is_aves&is_repro&atlas_isna,'code_atlas'] = (data[is_aves&is_repro&atlas_isna].statut_bio
.replace(['Reproduction','Nicheur'],'Nidification')
.replace(dict_label))
# définition du code atlas par le statut repro
atlas_isna = data.code_atlas.isna()
dict_repro = {
**dict.fromkeys(['couvaison','couveur','ois_couvaison'],'18'),
**dict.fromkeys(['accouplement','ois_accouplement','parade'],'6'),
**dict.fromkeys(['alarme'],'8'),
**dict.fromkeys(['chant','cht','ois_chant'],'3'),
**dict.fromkeys(['nourrissage','nourrissage des jeunes'],'16'),
**dict.fromkeys(['ois_nid avec jeune(s)'],'19'),
**dict.fromkeys(['ois_reproduction certaine (patrim)'],'50'),
**dict.fromkeys(['ois_reproduction possible'],'30'),
**dict.fromkeys(['transport matériaux'],'10'),
}
for k,v in dict_repro.items():
is_repro = data.comportement.str.lower()==k
if any(is_repro&atlas_isna):
data.loc[is_repro&is_aves&atlas_isna,'code_atlas'] = dict_code[v]
# Suppression des codes atlas érronés
is_dead = data.etat_bio=='mort'
atlas_notna = data.code_atlas.notna()
if any(is_dead&atlas_notna):
data.loc[is_dead&atlas_notna,'code_atlas'] = None
def form_observator(data):
return data.observateurs_v2.replace({
' \(CEN Isère\) \(CEN Isère\)':'',
' \(GRPLS\) \(GRPLS\)':'',
' \(CD Isère\) \(CD Isère\)':'',
' \(BIOTOPE\) \(BIOTOPE\)':'',
' \(Ecosphère\) \(Ecosphère\)':'',
' \(FMBDS\) \(FMBDS\)':'',
' \(Lo Parvi\) \(Lo Parvi\)':'',
' \(LPO Rhône\) \(LPO Rhône\)':'',
' \(Personnel\) \(Personnel\)':'',
' \(ONCFS\) \(ONCFS\)':'',
' \(Nature et Vie Sociale\) \(Nature et Vie Sociale\)':'',
' \(Conservatoire d’Espaces Naturels Isère\) \(Conservatoire d’Espaces Naturels Isère\)':'',
' \(Société Batrachologique de France\) \(Société Batrachologique de France\)':'',
'CEN Isère':'Conservatoire d’Espaces Naturels Isère',
},regex=True)
def form_coord(data):
t1 = data.longitude_x > 10
t2 = data.latitude_y > 50
data.loc[t1,'longitude_x'] = data[t1].to_crs(epsg=4326).geom.x
data.loc[t2,'latitude_y'] = data[t2].to_crs(epsg=4326).geom.y
def form_rmqobs(data):
data.rmq_observation.replace({
'Commentaires : Détails :':'Détails :',
'Commentaires : , localisation :':'localisation :',
},inplace=True,regex=True)
def recup_determ(data):
determ_isna = data.determination.isna()
is_statut = data.rmq_observation.str.contains('Indice de présence')
if is_statut.any():
data.loc[determ_isna&is_statut,'determination'] = 'Indice de présence'
data.loc[determ_isna&is_statut,'preuve_detrm'] = (data[determ_isna&is_statut].rmq_observation
.str.split('Indice de présence')
.str[1]
.str.split('|')
.str[0]
.str.strip()
.str.extract('.*\((.*)\).*')[0]
)
determ_isna = data.determination.isna()
data.loc[determ_isna,'determination'] = (data[determ_isna].rmq_observation
.str.extract('.*\((en main|observation indirecte|vu|contact auditif|analyse de pelotes)\).*')[0]
)
# determ_isna = data.determination.isna()
# in_hand = data.rmq_observation.str.contains('\(en main\)')
# if any(determ_isna&in_hand):
# data.loc[determ_isna&in_hand,'determination'] = 'en main'
determ_isna = data.determination.isna()
vu = data.rmq_observation.str.contains('Individu vu|Vu vivant|Commentaires \: Vu')
if any(determ_isna&vu):
data.loc[determ_isna&vu,'determination'] = 'Individu vu'
determ_isna = data.determination.isna()
entendu = data.rmq_observation.str.contains('Individu entendu')
if any(determ_isna&entendu):
data.loc[determ_isna&entendu,'determination'] = 'Individu entendu'
determ_isna = data.determination.isna()
photo = data.rmq_observation.str.contains('piège photo')
if any(determ_isna&photo):
data.loc[determ_isna&photo,'determination'] = 'Observation par piège photographique'
def recup_statutbio(data):
data['statut_bio'] = None
is_stat1 = data.rmq_observation.str.contains('Caractéristique',na=False)
if is_stat1.any():
statbio_isna = data.statut_bio.isna()
data.loc[statbio_isna&is_stat1,'statut_bio'] = (data[statbio_isna&is_stat1].rmq_observation
.str.split('Caractéristique :')
.str[1]
.str.split('\|')
.str[0]
.str.strip()
)
is_stat2 = data.rmq_observation.str.contains('Statut',na=False)
if is_stat2.any():
statbio_isna = data.statut_bio.isna()
data.loc[statbio_isna&is_stat2,'statut_bio'] = (data[statbio_isna&is_stat2].rmq_observation
.str.split('Statut')
.str[1]
.str.strip()
.str.removeprefix(':').str.strip()
.str.split('\||/|;')
.str[0]
.str.strip()
.str.extract('(Migrateur|Estivage|Reproduction probable|Reproduction certaine|Reproduction possible|Hivernant)')[0]
)
is_stat3 = data.rmq_observation.str.contains('Critere',na=False)
if is_stat3.any():
statbio_isna = data.statut_bio.isna()
data.loc[statbio_isna&is_stat3,'statut_bio'] = (data[statbio_isna&is_stat3].rmq_observation
.str.split('Critere')
.str[1]
.str.strip()
.str.removeprefix(':').str.strip()
.str.split('\||/|;')
.str[0]
.str.strip()
.str.extract('(nicheurs possibles|reproducteur)')[0]
)
is_stat4 = data.rmq_observation.str.contains('Commentaires',na=False)
if is_stat4.any():
statbio_isna = data.statut_bio.isna()
data.loc[statbio_isna&is_stat4,'statut_bio'] = (data[statbio_isna&is_stat4].rmq_observation
.str.split('Commentaires')
.str[1]
.str.strip()
.str.removeprefix(':').str.strip()
.str.lower()
.str.extract('(estivant|nicheur certain|nicheur probable|nicheur|semble nicher|niche|Reproduction probable|Reproduction certaine|Reproduction possible)')[0]
)
def recup_etatbio(data):
data['etat_bio'] = None
# Récupération de la mortalité dans le champs rmq_observation
is_dead1 = (data.rmq_observation
.str.lower()
.str.contains('individu mort|trouvé mort|mort : oui',na=False))
if is_dead1.any():
data.loc[is_dead1,'etat_bio'] = 'mort'
is_dead2 = data.comportement.str.lower()=='cadavre'
if is_dead2.any():
data.loc[is_dead2,'etat_bio'] = 'mort'
# nettoyage des comportements si individu mort
is_dead = is_dead1|is_dead2
compt_notna = data.comportement.notna()
if any(is_dead&compt_notna):
data.loc[is_dead&compt_notna,'comportement'] = None
# adaptation de la détermination
detm_hand = data.determination=='en main'
if any(detm_hand&is_dead):
data.loc[detm_hand&is_dead,'comportement'] = 'cadavre en main'
def recup_sexefaune(data):
is_male = data.rmq_observation.replace({'[x.0s]':''},regex=True).str.lower().str.contains('\\d mâle',na=False)
is_feml = data.rmq_observation.replace({'[x.0s]':''},regex=True).str.lower().str.contains('\\d femelle',na=False)
sexe_isna = data.sexe_faune.isna()
data.loc[sexe_isna&is_male&~is_feml,'sexe_faune'] = 'mâle'
data.loc[sexe_isna&~is_male&is_feml,'sexe_faune'] = 'femelle'
data.loc[sexe_isna&is_male&is_feml,'sexe_faune'] = 'mixte'
def format_faune(data):
recup_sexefaune(data)
recup_comptmt(data)
recup_etatbio(data)
recup_statutbio(data)
recup_atlas(data)
res = recup_stadevie(data)
return complete_objdenomb(res)
def format_data(data):
form_rmqobs(data)
data.observateurs_v2 = form_observator(data)
form_precision(data)
form_effectif(data)
correct_taxonomie(data)
form_cdnom(data)
form_date(data)
form_coord(data)
form_complx_grp(data)
recup_determ(data)
return format_faune(data)
def _export(path_name, data, format='csv'):
if format=='xlsx':
data.to_excel(path_name+'.%s'%format)
elif format=='csv':
data.to_csv(path_name+'.%s'%format)
else:
raise('format non prévu')
def _filter_diffusable(path_name,data):
if any(data.diffusable=='non'):
no_diff = data[data.diffusable=='non'].copy()
diff = data[data.diffusable=='oui'].copy()
to_export = {
path_name+'_diff':diff,
path_name+'_no_diff':no_diff
}
else:
to_export = {
path_name:data
}
return to_export
def _filter_uuid_sinp(to_export):
res = {}
for k,v in to_export.items():
if 'unique_id_sinp' in v.columns:
uuid_sinp = v[v.unique_id_sinp.notna()]
not_uuid = v[v.unique_id_sinp.isna()]
res |= {
k+'_uuid':uuid_sinp,
k+'_not_uuid':not_uuid
}
else:
res |= {
k:v
}
return res
def export(path_name, data, format='csv'):
# Formatage des dates avant écriture
detect_date = data.columns[data.columns.str.startswith('date')]
data.loc[:,detect_date] = data[detect_date].astype(str)
# Suppression de la colonne geom
df = data.to_wkt().drop(columns='geom',errors='ignore')
# Export
_to_export = _filter_diffusable(path_name,df)
to_export = _filter_uuid_sinp(_to_export)
for k,v in to_export.items():
if v.empty:
continue
_export(k,v,format)
def serena_rnngl_site():
from sqlalchemy import create_engine # pour lecture de la bd
from sqlalchemy.engine import URL
from shapely.geometry import Polygon
usr = 'postgres'
pdw = 'postgres'
bdd = 'serenadb'
host = '172.17.0.2'
eng = URL.create('postgresql+psycopg2',username=usr,password=pdw,host=host,database=bdd)
conn = create_engine(eng)
sit = gpd.pd.read_sql_table('rnf_site',con=conn,schema='serenabase')
sit['poly'] = (sit
.site_poly.str[9:]
.str.split(','))
sit.loc[sit.poly.notna(),'geom'] = (sit.loc[sit.poly.notna(),'poly']
.apply(lambda x: [xx.split(' ') for xx in x if xx])
.apply(lambda x: [[float(xxx) for xxx in xx] for xx in x ])
.apply(lambda x: Polygon(x)))
return (sit
.set_geometry('geom',crs=4326)
.to_crs(2154)
.dropna(subset=['geom']))
if __name__ == "__main__":
v_synthese_invertebre = 'v_synthese_invertebre'
v_synthese_vertebre = 'v_synthese_vertebre'
v_synthese_flore = 'v_synthese_flore'
sit_rnngl = serena_rnngl_site()
# sql_exclude_rnngl = " (rmq_localisation NOT ILIKE '%%lemps%%' OR NOT ST_INTERSECTS(geom,'SRID=2154;{}'))".format(sit_rnngl.unary_union)
sql_exclude_rnngl = " rmq_localisation NOT ILIKE '%%grand%%lemps%%'"
sql_rmqloc_null = " rmq_localisation IS NULL"
sql_inv = "SELECT * FROM saisie.%s WHERE cd_nom <> '9999081'"%v_synthese_invertebre # 9999081 : Heterocera sp.
v_inv = gpd.read_postgis(sql_inv+" AND"+sql_exclude_rnngl,con_sicen).dropna(how='all',axis=1)
v_inv_autre = gpd.read_postgis(sql_inv+" AND"+sql_rmqloc_null,con_sicen).dropna(how='all',axis=1)
v_inv = gpd.pd.concat([v_inv,v_inv_autre],ignore_index=True).reset_index(drop=True)
v_inv_rnngl = gpd.read_postgis(sql_inv+" AND"+sql_exclude_rnngl.replace('NOT ',''),con_sicen).dropna(how='all',axis=1)
t1_inv = v_inv_rnngl.etude.str.contains('Haute Bourbre 2014')
t2_inv = v_inv_rnngl.observateurs_v2.isin(['ZARADZKI Lise (CEN Isère)','BIRON Nicolas (CEN Isère)'])
t3_inv = v_inv.observateurs_v2=='LUCAS Jérémie (CEN Isère)'
t4_inv = v_inv.etude!='Echange de données - GRPLS'
hb_inv = v_inv_rnngl[t1_inv|t2_inv].copy()
no_stg = v_inv[t3_inv&t4_inv].copy()
v_inv = gpd.pd.concat([v_inv,hb_inv],ignore_index=True).drop_duplicates()
v_inv_rnngl.drop(hb_inv.index,inplace=True)
v_inv_rnngl = gpd.pd.concat([v_inv_rnngl,no_stg],ignore_index=True).drop_duplicates()
v_inv.drop(no_stg.index,inplace=True)
#v_synthese_invertebre = 37915
#v_inv.shape[0]+v_inv_rnngl.shape[0] = 37914
#différence de 1 donnée. Correspond au taxon 9999081 dans la bdd
sql_ver = "SELECT * FROM saisie.%s WHERE cd_nom <> '9999056'"%v_synthese_vertebre # 9999056 : Micromammalia sp. (17 données)
v_ver = gpd.read_postgis(sql_ver+" AND"+sql_exclude_rnngl,con_sicen).dropna(how='all',axis=1)
v_ver_autre = gpd.read_postgis(sql_ver+" AND"+sql_rmqloc_null,con_sicen).dropna(how='all',axis=1)
v_ver = gpd.pd.concat([v_ver,v_ver_autre],ignore_index=True).reset_index(drop=True)
v_ver_rnngl = gpd.read_postgis(sql_ver+" AND"+sql_exclude_rnngl.replace('NOT ',''),con_sicen).dropna(how='all',axis=1)
t1_ver = v_ver_rnngl.etude.str.contains('Haute Bourbre 2014')
t2_ver = v_ver_rnngl.observateurs_v2.isin(['ZARADZKI Lise (CEN Isère)','BIRON Nicolas (CEN Isère)'])
t3_ver = v_ver.observateurs_v2.isin(['LUCAS Jérémie (CEN Isère)','PRUNIER Jérôme (Personnel)', 'MAILLET Grégory (CEN Isère)','LAFON Arnaud (CEN Isère)'])
t4_ver = v_ver.etude!='Echange de données - GRPLS'
hb_ver = v_ver_rnngl[t1_ver|t2_ver].copy()
sg_ver = v_ver[t3_ver&t4_ver].copy()
v_ver = gpd.pd.concat([v_ver,hb_ver],ignore_index=True).drop_duplicates()
v_ver_rnngl.drop(hb_ver.index,inplace=True)
v_ver_rnngl = gpd.pd.concat([v_ver_rnngl,sg_ver],ignore_index=True).drop_duplicates()
v_ver.drop(sg_ver.index,inplace=True)
#v_synthese_vertebre = 102175
#v_ver.shape[0]+v_ver_rnngl.shape[0] = 102158
#différence de 17 donnée. Correspond au taxon 9999056 dans la bdd
sql_flo = 'SELECT * FROM saisie.%s'%v_synthese_flore
v_flo = gpd.read_postgis(sql_flo+" WHERE"+sql_exclude_rnngl,con_sicen).dropna(how='all',axis=1)
v_flo_autre = gpd.read_postgis(sql_flo+" WHERE"+sql_rmqloc_null,con_sicen).dropna(how='all',axis=1)
v_flo = gpd.pd.concat([v_flo,v_flo_autre],ignore_index=True).reset_index(drop=True)
v_flo_rnngl = gpd.read_postgis(sql_flo+" WHERE"+sql_exclude_rnngl.replace('NOT ',''),con_sicen).dropna(how='all',axis=1)
t1_flo = v_flo_rnngl.etude.str.contains('Haute Bourbre 2014')
t2_flo = v_flo_rnngl.observateurs_v2=='ZARADZKI Lise (CEN Isère)'
t3_flo = v_flo.observateurs_v2=='LUCAS Jérémie (CEN Isère)'
t4_flo = v_flo.etude!='Echange de données - GRPLS'
hb_flo = v_flo_rnngl[t1_flo|t2_flo].copy()
sg_flo = v_flo[t3_flo&t4_flo].copy()
v_flo = gpd.pd.concat([v_flo,hb_flo],ignore_index=True).drop_duplicates()
v_flo_rnngl.drop(hb_flo.index,inplace=True)
idflo_drop = v_flo[v_flo.id_obs.isin(v_flo_rnngl.id_obs)].index
v_flo.drop(idflo_drop,inplace=True)
v_flo_rnngl = gpd.pd.concat([v_flo_rnngl,sg_flo],ignore_index=True).drop_duplicates()
v_flo.drop(sg_flo.index,inplace=True)
#v_synthese_flore = 113556
#v_flo.shape[0]+v_flo_rnngl.shape[0] = 113556
v_rnngl = gpd.pd.concat([v_ver_rnngl,v_flo_rnngl,v_inv_rnngl],ignore_index=True)
v_inv_explode = format_data(v_inv)
V_VER = v_ver.copy()
v_ver = V_VER.copy()
v_ver_explode = format_data(v_ver)
format_data(v_flo)
inv_uuid = (gpd.pd.read_csv(os.path.join(DIR,'INVERTEBRE/','ORB_UUID','donnees_importees_cen38.csv'),sep=';')
.rename(columns={'entity_source_pk_value':'id_obs'}))
v_inv_explode = v_inv_explode.merge(inv_uuid[['id_obs', 'unique_id_sinp']],how='left',on='id_obs')
# list_inv = v_inv_explode[v_inv_explode.unique_id_sinp.notna()].etude.unique()
# INVERTEBRE
for etude in v_inv_explode.etude.unique():
exp_inv = v_inv_explode[v_inv_explode.etude==etude].copy()
exp_inv.dropna(how='all',inplace=True,axis=1)
if 'complexe_groupe' in exp_inv.columns:
exp_inv1 = exp_inv[exp_inv.complexe_groupe.notna()]
exp_inv2 = exp_inv[exp_inv.complexe_groupe.isna()].dropna(how='all',axis=1)
export(os.path.join(DIR,'INVERTEBRE','INV_'+etude+'_GRP'),exp_inv1,format='csv')
export(os.path.join(DIR,'INVERTEBRE','INV_'+etude),exp_inv2,format='csv')
else :
export(os.path.join(DIR,'INVERTEBRE','INV_'+etude),exp_inv,format='csv')
# VERTEBRE
for etude in v_ver_explode.etude.unique():
exp_ver = v_ver_explode[v_ver_explode.etude==etude].copy()
exp_ver.dropna(how='all',inplace=True,axis=1)
if 'complexe_groupe' in exp_ver.columns:
exp_ver1 = exp_ver[exp_ver.complexe_groupe.notna()]
exp_ver2 = exp_ver[exp_ver.complexe_groupe.isna()].dropna(how='all',axis=1)
export(os.path.join(DIR,'VERTEBRE','VER_'+etude+'_GRP'),exp_ver1,format='csv')
export(os.path.join(DIR,'VERTEBRE','VER_'+etude),exp_ver2,format='csv')
else :
export(os.path.join(DIR,'VERTEBRE','VER_'+etude),exp_ver,format='csv')
# FLORE
for etude in v_flo.etude.unique():
exp_flo = v_flo[v_flo.etude==etude].copy()
exp_flo.dropna(how='all',inplace=True,axis=1)
if 'complexe_groupe' in exp_flo.columns:
exp_flo1 = exp_flo[exp_flo.complexe_groupe.notna()]
exp_flo2 = exp_flo[exp_flo.complexe_groupe.isna()].dropna(how='all',axis=1)
export(os.path.join(DIR,'FLORE','FLO_'+etude+'_locNULL'+'_GRP'),exp_flo1,format='csv')
export(os.path.join(DIR,'FLORE','FLO_'+etude+'_locNULL'),exp_flo2,format='csv')
else :
export(os.path.join(DIR,'FLORE','FLO_'+etude+'_locNULL'),exp_flo,format='csv')
# _export(os.path.join(DIR,v_synthese_invertebre+'2'),v_inv_explode.dropna(how='all',axis=1))
# _export(os.path.join(DIR,v_synthese_invertebre),v_inv.dropna(how='all',axis=1))
# _export(os.path.join(DIR,v_synthese_vertebre+'2'),v_ver_explode.dropna(how='all',axis=1))
# _export(os.path.join(DIR,v_synthese_vertebre),v_ver.dropna(how='all',axis=1))
# _export(os.path.join(DIR,v_synthese_flore),v_flo)
# v_ver.etude.unique()
# v_ver.protocole.unique()
# v_ver.lot_donnee.unique()
1_SICEN/SERENA_siege/recovery_geo.py
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
from pycen import con_sicen, update_to_sql
import pandas as pd
import geopandas as gpd
from os import path
def get_site_serena():
sql = '''
SELECT s.*,
t.*,
c.choi_nom
FROM serenabase.rnf_site s
JOIN serenabase.tmp_sgll t ON t.sgll_site_id = s.site_id
JOIN serenabase.rnf_choi c ON c.choi_id = s.site_categ_choi_id
'''
return pd.read_sql_query(sql,con_sicen).dropna(axis=1,how='all')
if __name__ == "__main__":
# Lecture des sites SIG
PATH = '/media/cgeier/SRV/FICHIERS/OUTILS/BASES DE DONNEES/RHOMEO/2018/RhoMéO_Données'
FILE = '38_sig_points_de_suivis_point.shp'
sig = gpd.read_file(path.join(PATH,FILE))
# Récupération des sites SERENA
site_serena = get_site_serena()
no_coord = site_serena.sgll_lat=='999'
is_rhomeo = site_serena.site_nom.str[-3:].str.isnumeric()
is_rhomeo2 = site_serena.choi_nom.str.contains('rhomeo',case=False)
# site_serena[no_coord]
# site_serena[is_rhomeo]
# Isolation des sites RHOMEO
site_rhomeo_all = site_serena[is_rhomeo2&no_coord].copy()
site_rhomeo = site_rhomeo_all.merge(sig[['id_regio','geometry']],left_on='site_ref_sig',right_on='id_regio')\
.set_geometry('geometry',crs=2154)
site_rhomeo.to_crs(4326,inplace=True)
site_rhomeo.sgll_lon = site_rhomeo.geometry.x.astype(str)
site_rhomeo.sgll_lat = site_rhomeo.geometry.y.astype(str)
updt_site = site_rhomeo[['sgll_site_id','sgll_lat','sgll_lon']].copy()
# Update data
update_to_sql(
df=updt_site,
con=con_sicen,
table_name='tmp_sgll',
schema_name='serenabase',
key_name='sgll_site_id'
)
1_SICEN/SERENA_siege/recovery_relv.py
+190
View File
@@ -0,0 +1,190 @@
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import pandas_access as mdb # pour lecture de la bd medwet
from os import path
from pycen import con_sicen, update_to_sql
import pandas as pd
# access / postgis
SOURCE = 'postgis'
postgis_sch = 'serenabase'
DICT_RLV_COLS = {
'relv_id':'id_etude',
'relv_nom':'nom_etude',
'relv_prop_libel':'proprietaire',
'relv_1date_c':'date_debut',
'relv_2date_c':'date_fin',
'relv_comment':'description',
}
DICT_OBS_COLS = {
'obse_id':'id_origine',
'obse_relv_id':'id_etude',
}
def list_access_datatable():
PATH_IN = '/media/cgeier/SRV/FICHIERS/OUTILS/BASES DE DONNEES/ESPECES/backup'
BDD_IN = 'AVENIR.serena_V2.mdb'
LST_SCHEMA = mdb.read_schema(path.join(PATH_IN,BDD_IN))
LST_TABLE = [ *LST_SCHEMA.keys() ]
LST_TABLE = mdb.list_tables(path.join(PATH_IN,BDD_IN))
TABLE_DATA= []
for TABLE in LST_TABLE:
DTYPE = {x : float for x in LST_SCHEMA[TABLE].keys() if LST_SCHEMA[TABLE][x]=='Long Integer'}
df = mdb.read_table(path.join(PATH_IN,BDD_IN),TABLE, dtype=DTYPE)
if df.shape[0] == 0:
print('%s IS EMPTY !'%TABLE)
else :
TABLE_DATA += [TABLE]
return TABLE_DATA
def list_releve(source='postgis'):
if source=='postgis':
choi = pd.read_sql_table('rnf_choi',con_sicen,postgis_sch)\
.dropna(axis=1,how='all')
relv = pd.read_sql_table('rnf_relv',con_sicen,postgis_sch)\
.dropna(axis=1,how='all')
# Correction coquille
relv.loc[relv.relv_categ_choi_id==100800,'relv_categ_choi_id'] = 100008
# Jointure relevé / catégorie
df = relv.merge(choi[['choi_id','choi_nom']],how='inner',left_on='relv_categ_choi_id',right_on='choi_id')\
.drop(columns=['relv_categ_choi_id'])
# Mise en forme des dates
lstcol_dat = df.columns[df.columns.str.contains('date')]
for lcd in lstcol_dat:
# df[lcd] = df[lcd].replace({'VIDE':None})
df.loc[df[lcd]=='2000 à 2001',lcd] = '2000' if '1date' in lcd else '2001'
df[lcd].replace({'VIDE':None},inplace=True)
df[lcd] = pd.to_datetime(df[lcd])
return df
def get_serena_obs(source='postgis'):
if source=='postgis':
obs = pd.read_sql_table('rnf_obse',con_sicen,postgis_sch)
return obs
def get_sicen_obs():
return pd.read_sql_table('saisie_observation',con_sicen,'saisie')
if __name__ == "__main__":
if SOURCE == 'access':
lst_tab = list_access_datatable()
elif SOURCE == 'postgis':
lstrelv_serena = list_releve()
obs_serena = get_serena_obs()
obs_serena = obs_serena.merge(lstrelv_serena[['relv_id','relv_nom']],left_on='obse_relv_id',right_on='relv_id')\
.drop(columns='relv_id')
# Isolation des relevés HORS RhoMéo
is_relv = [x for x in lstrelv_serena.relv_nom.str[:3].unique() if not x.isnumeric()]
relv_serena = lstrelv_serena[lstrelv_serena.relv_nom.str[:3].isin(is_relv)]
# Formatage des relevés HORS RhoMéo
relv_2sicen = relv_serena.rename(columns=DICT_RLV_COLS).filter(DICT_RLV_COLS.values(),axis=1)
relv_2sicen.sort_values('id_etude',inplace=True)
relv_2sicen.reset_index(inplace=True,drop=True)
relv_2sicen.set_index(-relv_2sicen.index,inplace=True)
# # Intégration des relevés SERENA dans la table 'etude'
# relv_2sicen.drop(columns=['id_etude','proprietaire'])\
# .rename_axis('id_etude')\
# .to_sql('etude',con_sicen,'md',if_exists='append')
# Récupération des données SICEN2
# id_lot = 'SERENA'
sicen_obs = get_sicen_obs()
is_serena = sicen_obs.id_lot==3
is_horsetude = sicen_obs.id_etude==2
lst_idorigine = sicen_obs[is_serena].id_origine.astype(int).tolist()
# isolation des données "suposées" non présentes
obs_notinsicen = obs_serena[~obs_serena.obse_id.isin(lst_idorigine)].copy()
# rhomeo_notinsicen = obs_notinsicen.obse_nom.str.startswith('RhoMéO',na=False)
# obs_notinsicen[rhomeo_notinsicen]
# test = obs_notinsicen.obse_site_id == 101335
# obs_notinsicen[test]
sql1 = "SELECT * FROM {sch}.tmp_ogll WHERE ogll_obse_id IN {lst} AND ogll_lat <> '999'"
res1 = pd.read_sql_query(sql1.format(sch=postgis_sch,lst=tuple(obs_notinsicen.obse_id)),con_sicen)
sql2 = "SELECT * FROM {sch}.tmp_sgll WHERE sgll_site_id IN {lst} AND sgll_lat <> '999'"
res2 = pd.read_sql_query(sql2.format(sch=postgis_sch,lst=tuple(obs_notinsicen.obse_site_id.unique())),con_sicen)
miss_serena1 = obs_notinsicen.merge(
res1.rename(columns={
'ogll_obse_id':'obse_id',
'ogll_lat':'lat',
'ogll_lon':'lon'
}),
on='obse_id')
miss_serena2 = obs_notinsicen.merge(
res2.rename(columns={
'sgll_site_id':'obse_site_id',
'sgll_lat':'lat',
'sgll_lon':'lon'
}),
on='obse_site_id')
miss_serena = pd.concat([miss_serena1,miss_serena2])
rhomeo_missserena = miss_serena.obse_nom.str.startswith('RhoMéO',na=False)
miss_serena[rhomeo_missserena]
# Formatages des données OBS
obs_2sicen = obs_serena[obs_serena.obse_relv_id.isin(relv_serena.relv_id)].rename(columns=DICT_OBS_COLS).filter(DICT_OBS_COLS.values(),axis=1)
obs_2sicen['id_lot'] = 3
obs_2sicen.id_origine = obs_2sicen.id_origine.astype(str)
# Correspondance id_relv (SERENA) / id_etude (SICEN2)
DICT_ID_ETUDE = dict(zip(relv_2sicen.id_etude,relv_2sicen.index))
obs_2sicen.id_etude.replace(DICT_ID_ETUDE,inplace=True)
# Mise à jour du champs id_etude de la table saisie_observation
update_to_sql(
obs_2sicen[obs_2sicen.id_origine.astype(int).isin(lst_idorigine)],
con_sicen,
'saisie_observation',
'saisie',
['id_lot','id_origine']
)
# identification des données restantes taguées "hors etude"
id_origine_he = sicen_obs[is_serena&is_horsetude].id_origine.astype(int)
is_horsetude2 = obs_serena.obse_id.isin(id_origine_he)
res_he = obs_serena[is_horsetude2].copy()
DICT2_OBS_COLS = {**DICT_OBS_COLS,'relv_nom':'id_waypoint'}
obs2_2sicen = res_he.rename(columns=DICT2_OBS_COLS).filter(DICT2_OBS_COLS.values(),axis=1)
obs2_2sicen['id_lot'] = 3
obs2_2sicen['id_etude'] = -152
obs2_2sicen.id_origine = obs2_2sicen.id_origine.astype(str)
# Mise à jour du champs id_etude de la table saisie_observation
update_to_sql(
obs2_2sicen,
con_sicen,
'saisie_observation',
'saisie',
['id_lot','id_origine']
)
1_SICEN/SERENA_siege/recovery_rhomeo.py
+464
View File
@@ -0,0 +1,464 @@
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import pandas_access as mdb
from pycen import con_sicen, update_to_sql
import pandas as pd
import geopandas as gpd
from os import path
postgis_sch = 'serenabase'
ACCESS_PATH = '/media/cgeier/SRV/FICHIERS/OUTILS/BASES DE DONNEES/RHOMEO/2012/CEN-38_RHOMEO_BDD_2012'
ACCESS_FILE = 'BD_Rhomeo_data.mdb'
DICT_COLS = {
'obse_id':'id_origine',
}
def get_columns_oftable(tab,sch,con):
res = con.dialect.get_columns(con,tab,sch)
return [x['name'] for x in res]
def get_columns_dtypes(tab,sch,con,df:pd.DataFrame=None):
res = con.dialect.get_columns(con,tab,sch)
if isinstance(df,pd.DataFrame):
return {x['name']:x['type'] for x in res if x['name'] in df.columns}
else:
return {x['name']:x['type'] for x in res}
def match_columns(df,tab,sch,con):
cols_saisie = get_columns_oftable(tab,sch,con)
obs_cols = df.columns
return obs_cols[obs_cols.isin(cols_saisie)]
def lower(df):
for c in df.columns :
df[c] = df[c].str.lower() if df[c].apply(type).eq(str).all() else df[c]
return df
def get_odat_obse(id_obse:gpd.np.ndarray|list|pd.Series|int=None):
sql = '''
SELECT
o.odat_obse_id,
c.choi_nom,
l.list_nom,
o.odat_nom
FROM {sch}.rnf_odat o
LEFT JOIN {sch}.rnf_choi c ON c.choi_id = o.odat_choi_id
LEFT JOIN {sch}.rnf_list l ON l.list_id = o.odat_c_list_id
'''.format(sch=postgis_sch)
if isinstance(id_obse,int) :
sql += ' WHERE odat_obse_id = %i'%id_obse
elif isinstance(id_obse,(list,gpd.pd.Series,gpd.np.ndarray)) :
sql += ' WHERE odat_obse_id IN {lst_id}'.format(lst_id=tuple(id_obse))
result = pd.read_sql_query(sql,con_sicen,index_col=['odat_obse_id','list_nom']).replace({r' \(RhoMeO\)':'',r'\?':'',r'[.]':''},regex=True)
df = lower(result).pivot(columns='choi_nom',values='odat_nom').reset_index(drop=False)
return format_comportement(df)
def format_comportement(oda:pd.DataFrame):
DICT = {
'exuvie/émergent':'ODO_Exuvie/émergence',
'mâles+femelles':'ODO_Mâles+Femelles',
'autre':'ODO_Autre',
'territoriale':'ODO_Territorial',
'ponte':'ODO_Ponte',
'accouplement':'ODO_Tandem',
'tandem':'ODO_Tandem',
'aucun':None,
}
return oda.replace(DICT)
def format_odat(df,odat):
DICT_COLS = {
'comportement':'reprostatut',
'strate':'type_effectif',
'rmq_odat':'remarque_obs'
}
df_odat = df[['obse_id','obse_nombre','obse_nom','cd_nom','ordre']].merge(
odat,
left_on='obse_id',right_on='odat_obse_id'
)
nb_cols = odat.columns[odat.columns.str.startswith('nb')]
df_odat['rmq_odat'] = None
for c in nb_cols:
test1 = df_odat.obse_nombre!=df_odat[c]
test2 = df_odat[c].notna()
test3 = df_odat.rmq_odat.notna()
test = test1&test2&test3
tes = test1&test2&~test3
if any(tes) or any(test):
tt = '{rmq_o}; {c}:{data}'.format(rmq_o=df_odat[test].rmq_odat,c=c,data=df_odat[test][c])
df_odat.loc[test,'rmq_odat'] = df_odat[test].rmq_odat+';'+c+':'+df_odat[test][c]
df_odat.loc[tes,'rmq_odat'] = c+':'+df_odat[tes][c]
return df_odat.rename(columns=DICT_COLS)[['odat_obse_id',*DICT_COLS.values()]]
def get_obs_serena(insicen:bool=None):
sql = '''
SELECT o.*,
s.site_nom,
s.site_ref_sig,
CASE WHEN t.taxo_id = 203471 THEN (SELECT cd_nom FROM inpn.taxref WHERE cd_nom = '54213' )
WHEN t.taxo_id = 203491 THEN (SELECT cd_nom FROM inpn.taxref WHERE cd_nom = '521494' )
WHEN tax.cd_nom IS NULL THEN ta.cd_nom::text
ELSE tax.cd_nom
END cd_nom,
CASE WHEN t.taxo_id = 203471 THEN (SELECT phylum FROM inpn.taxref WHERE cd_nom = '54213' )
WHEN t.taxo_id = 203491 THEN (SELECT phylum FROM inpn.taxref WHERE cd_nom = '521494' )
WHEN tax.phylum IS NULL THEN ta.phylum
ELSE tax.phylum
END phylum,
CASE WHEN t.taxo_id = 203471 THEN (SELECT classe FROM inpn.taxref WHERE cd_nom = '54213' )
WHEN t.taxo_id = 203491 THEN (SELECT classe FROM inpn.taxref WHERE cd_nom = '521494' )
WHEN tax.classe IS NULL THEN ta.classe
ELSE tax.classe
END classe,
CASE WHEN t.taxo_id = 203471 THEN (SELECT ordre FROM inpn.taxref WHERE cd_nom = '54213' )
WHEN t.taxo_id = 203491 THEN (SELECT ordre FROM inpn.taxref WHERE cd_nom = '521494' )
WHEN tax.ordre IS NULL THEN ta.ordre
ELSE tax.ordre
END ordre,
CASE WHEN t.taxo_id = 203471 THEN (SELECT famille FROM inpn.taxref WHERE cd_nom = '54213' )
WHEN t.taxo_id = 203491 THEN (SELECT famille FROM inpn.taxref WHERE cd_nom = '521494' )
WHEN tax.famille IS NULL THEN ta.famille
ELSE tax.famille
END famille,
CASE WHEN t.taxo_id = 203471 THEN (SELECT nom_complet FROM inpn.taxref WHERE cd_nom = '54213' )
WHEN t.taxo_id = 203491 THEN (SELECT nom_complet FROM inpn.taxref WHERE cd_nom = '521494' )
WHEN tax.nom_complet IS NULL THEN ta.nom_complet
ELSE tax.nom_complet
END nom_complet,
CASE WHEN t.taxo_id = 203471 THEN (SELECT nom_vern FROM inpn.taxref WHERE cd_nom = '54213' )
WHEN t.taxo_id = 203491 THEN (SELECT nom_vern FROM inpn.taxref WHERE cd_nom = '521494' )
WHEN tax.nom_vern IS NULL THEN ta.nom_vern
ELSE tax.nom_vern
END nom_vern,
CASE WHEN tmpo.ogll_lon = '999' THEN tmps.sgll_lon::float
ELSE tmpo.ogll_lon::float
END lon,
CASE WHEN tmpo.ogll_lat = '999' THEN tmps.sgll_lat::float
ELSE tmpo.ogll_lat::float
END lat
FROM {sch}.rnf_obse o
LEFT JOIN {sch}.rnf_relv r ON r.relv_id = o.obse_relv_id
LEFT JOIN {sch}.rnf_site s ON s.site_id = o.obse_site_id
LEFT JOIN {sch}.tmp_sgll tmps ON tmps.sgll_site_id = o.obse_site_id
LEFT JOIN {sch}.tmp_ogll tmpo ON tmpo.ogll_obse_id = o.obse_id
LEFT JOIN serenarefe.rnf_taxo t ON t.taxo_id = o.obse_taxo_id
LEFT JOIN inpn.taxons_isere_absents_taxref ta ON ta.id_taxon = o.obse_taxo_id
LEFT JOIN inpn.taxref tax ON t.taxo_mnhn_id = tax.cd_nom::int
--LEFT JOIN {sch}.rnf_odat od ON od.odat_obse_id = o.obse_id
'''.format(sch=postgis_sch)
if insicen is not None :
sql_sicen = 'SELECT DISTINCT id_origine FROM saisie.saisie_observation WHERE id_lot=3'
sicen_obs = pd.read_sql_query(sql_sicen,con_sicen)
if insicen is True :
sql += 'WHERE obse_id IN {lst_id}'.format(lst_id=tuple(sicen_obs.id_origine.astype(int)))
if insicen is False :
sql += 'WHERE obse_id NOT IN {lst_id}'.format(lst_id=tuple(sicen_obs.id_origine.astype(int)))
return pd.read_sql_query(sql,con_sicen)\
.dropna(axis=1,how='all')
def list_releve(id_relv=None,source='postgis'):
if source=='postgis':
df_choi = pd.read_sql_table('rnf_choi',con_sicen,postgis_sch)\
.dropna(axis=1,how='all')
df_relv = pd.read_sql_table('rnf_relv',con_sicen,postgis_sch)\
.dropna(axis=1,how='all')
# Correction coquille
df_relv.loc[df_relv.relv_categ_choi_id==100800,'relv_categ_choi_id'] = 100008
# Jointure relevé / catégorie
df = df_relv.merge(df_choi[['choi_id','choi_nom']],how='inner',left_on='relv_categ_choi_id',right_on='choi_id')\
.drop(columns=['relv_categ_choi_id'])
# Mise en forme des dates
lstcol_dat = df.columns[df.columns.str.contains('date')]
for lcd in lstcol_dat:
# df[lcd] = df[lcd].replace({'VIDE':None})
df.loc[df[lcd]=='2000 à 2001',lcd] = '2000' if '1date' in lcd else '2001'
df[lcd].replace({'VIDE':None},inplace=True)
df[lcd] = pd.to_datetime(df[lcd])
if id_relv is not None:
if isinstance(id_relv,int) : filtre = df.relv_id == id_relv
elif isinstance(id_relv,(list,gpd.pd.Series,gpd.np.ndarray)) : filtre = df.relv_id.isin([*id_relv])
return df[filtre]
else:
return df
def get_serena_obs(srce_id=None,source:str='access'):
DICT_STR = {
'avenir':'cen isère',
'be':'personnel'
}
if source == 'postgis':
sql = 'SELECT user_id,user_srce_id FROM {sch}.rnf_user'.format(sch=postgis_sch)
if srce_id is not None:
if isinstance(srce_id,int) :
sql += ' WHERE user_srce_id = %i'%srce_id
elif isinstance(srce_id,(list,gpd.pd.Series,gpd.np.ndarray)) :
sql += ' WHERE user_srce_id IN {lst_id}'.format(lst_id=tuple(srce_id))
return lower(pd.read_sql_query(sql,con_sicen))
elif source == 'access':
df = mdb.read_table(path.join(ACCESS_PATH,ACCESS_FILE),'rev_obs')
if srce_id is not None:
if isinstance(srce_id,int) :
df = df[df.rev_observateur==srce_id]
elif isinstance(srce_id,(list,gpd.pd.Series,gpd.np.ndarray)) :
df = df[df.rev_observateur.isin([*srce_id])]
df[['nom','prenom']] = df.nom_observateur.str.split(' ',1,expand=True)
return lower(df).replace(DICT_STR)
def get_sicen_obs(personne_id=None):
sql = 'SELECT id_personne,prenom,nom,id_structure,nom_structure FROM md.personne JOIN md.structure USING (id_structure)'.format(sch=postgis_sch)
if personne_id is not None:
if isinstance(personne_id,int) :
sql += ' WHERE personne_id = %i'%personne_id
elif isinstance(personne_id,str) :
sql += ' WHERE personne_id = %i'%int(personne_id)
elif isinstance(personne_id,(list,gpd.pd.Series,gpd.np.ndarray)) :
sql += ' WHERE personne_id IN {lst_id}'.format(lst_id=tuple(personne_id))
return lower(pd.read_sql_query(sql,con_sicen))
def get_sicen_etude():
sql = 'SELECT * FROM md.etude'
return pd.read_sql_query(sql, con_sicen)
def get_serena_choi(choi_id:gpd.np.ndarray|list|pd.Series|int=None):
''''''
sql = 'SELECT choi_id,choi_nom FROM {sch}.rnf_choi'.format(sch=postgis_sch)
if isinstance(choi_id,int) :
sql += ' WHERE choi_id = %i'%choi_id
elif isinstance(choi_id,(list,gpd.pd.Series,gpd.np.ndarray)) :
sql += ' WHERE choi_id IN {lst_id}'.format(lst_id=tuple(choi_id))
return lower(pd.read_sql_query(sql,con_sicen))
def get_serena_site(site_id:gpd.np.ndarray|list|pd.Series|int=None):
sql = 'SELECT site_id,site_nom FROM {sch}.rnf_site'.format(sch=postgis_sch)
if isinstance(site_id,int) :
sql += ' WHERE site_id = %i'%site_id
elif isinstance(site_id,(list,gpd.pd.Series,gpd.np.ndarray)) :
sql += ' WHERE site_id IN {lst_id}'.format(lst_id=tuple(site_id))
return pd.read_sql_query(sql,con_sicen)
def get_sicen_pcol():
sql = '''SELECT * FROM "md"."protocole";'''
return lower(pd.read_sql_query(sql,con_sicen))
def _crsp(left:pd.DataFrame,right:pd.DataFrame,lefton,righton,idleft,idright):
df = left.merge(right,left_on=lefton,right_on=righton)
return dict(zip(df[idleft],df[idright]))
def crsp_relv(relv:pd.Series):
seren_relv = lower(list_releve(relv.unique()))
sicen_relv = lower(get_sicen_etude())
DICT_RELV_ID = _crsp(left=seren_relv,right=sicen_relv,lefton='relv_nom',righton='nom_etude',idleft='relv_id',idright='id_etude')
return relv.replace(DICT_RELV_ID)
def crsp_obs(obs:pd.Series):
obs_seren = get_serena_obs(obs.unique())
obs_sicen = get_sicen_obs()
DICT_OBS_ID = _crsp(left=obs_seren,right=obs_sicen,lefton=['nom','structure_obs'],righton=['nom','nom_structure'],idleft='rev_observateur',idright='id_personne')
return obs.replace(DICT_OBS_ID)
def crsp_pcol(pcol:pd.Series):
pcol_seren = get_serena_choi(pcol.unique())
pcol_sicen = get_sicen_pcol()
DICT_PCOL_ID = _crsp(left=pcol_seren,right=pcol_sicen,lefton='choi_nom',righton='libelle',idleft='choi_id',idright='id_protocole')
return pcol.replace(DICT_PCOL_ID)
def crsp_valid(valid:pd.Series):
pcol_seren = get_serena_choi(valid.unique())
pcol_sicen = get_sicen_pcol()
DICT_PCOL_ID = _crsp(left=pcol_seren,right=pcol_sicen,lefton='choi_nom',righton='libelle',idleft='choi_id',idright='id_protocole')
return valid.replace(DICT_PCOL_ID)
def crsp_abond(abond:pd.Series):
ab_seren = get_serena_choi(abond.dropna().unique())
DICT_ABOND_ID = dict(zip(ab_seren.choi_id,ab_seren.choi_nom))
return abond.replace(DICT_ABOND_ID)
def crsp_site(site:pd.Series):
sit_seren = get_serena_site(site.dropna().unique())
DICT_SITE_ID = dict(zip(sit_seren.site_id,sit_seren.site_nom))
return site.replace(DICT_SITE_ID)
def get_structure_id(id_pers:gpd.np.ndarray|list|pd.Series|int=None):
sql = 'SELECT id_personne, id_structure FROM md.personne'
if isinstance(id_pers,int) :
sql += ' WHERE id_personne = %i'%id_pers
elif isinstance(id_pers,(list,gpd.pd.Series,gpd.np.ndarray)) :
sql += ' WHERE id_personne IN {lst_id}'.format(lst_id=tuple(id_pers))
return pd.read_sql_query(sql,con_sicen)
def format_effectif(DF):
df = DF.copy()
isnum = df.effectif_min.str.isnumeric()
df.loc[isnum,'effectif'] = df[isnum].effectif_min.copy()
df.loc[isnum,'effectif_min'] = None
df.loc[~isnum,'effectif_textuel'] = df[~isnum].effectif_min.copy()
df.loc[~isnum,'effectif_min'] = df[~isnum].effectif_textuel.str.split(r'[-+]',expand=True)[0]
df.loc[~isnum,'effectif_max'] = df[~isnum].effectif_textuel.str.split(r'[-+]',expand=True)[1]
return df.replace({'':None})
def to_sicen(DF):
CRSP_COLUMNS = {
'obse_id':'id_origine',
'obse_relv_id':'id_etude',
'obse_obsv_id':'observateur',
'obse_detm_id':'validateur',
'obse_date':'date_obs',
# 'obse_site_id':'localisation',
'obse_pcole_choi_id':'id_protocole',
'obse_validat_choi_id':'statut_validation',
'obse_confid_choi_id':'diffusable',
'obse_abond_choi_id':'effectif_textuel',
'obse_nombre':'effectif_min',
'site_nom':'localisation',
'site_ref_sig':'id_waypoint',
'lat':'latitude',
'lon':'longitude',
}
df = DF.copy()
if df.geometry.name != 'geometrie' :
df.rename_geometry('geometrie',inplace=True)
df.obse_relv_id = crsp_relv(df.obse_relv_id)
df.obse_obsv_id = crsp_obs(df.obse_obsv_id)
df.obse_detm_id = crsp_obs(df.obse_detm_id).astype(int)
df.obse_date = pd.to_datetime(df.obse_date)
# df.obse_site_id = crsp_site(df.obse_site_id)
df.obse_pcole_choi_id = crsp_pcol(df.obse_pcole_choi_id)
df.loc[df.obse_validat_choi_id==100373,'obse_validat_choi_id'] = 'validée'
df.loc[df.obse_confid_choi_id==100473,'obse_confid_choi_id'] = True
df.obse_abond_choi_id = crsp_abond(df.obse_abond_choi_id)
df.rename(columns=CRSP_COLUMNS,inplace=True)
# Tag du lot SERENA
df['id_lot'] = 3
# Jointure des id_structure
struct = get_structure_id(df.observateur.unique())
df = df.merge(struct,how='left',left_on='observateur', right_on='id_personne')\
.rename(columns={'id_structure':'structure'})
cols = match_columns(df,'saisie_observation','saisie',con_sicen)
# Formatage des effectifs
return format_effectif(df[cols])
def drop_cdnom_missing(df):
isna = df.cd_nom.isna()
idx_na = df[isna].index
return df.drop(idx_na)
if __name__ == "__main__":
obs_serena = get_obs_serena(insicen=True)
isna = obs_serena.cd_nom.isna()
# Recréation des géométries
gdf_obs = obs_serena.set_geometry(gpd.points_from_xy(obs_serena.lon,obs_serena.lat))
gdf_obs.set_crs(4326,inplace=True)
gdf_obs.to_crs(2154,inplace=True)
# Identification des données RhoMeo
is_rhomeo = gdf_obs.obse_nom.str.contains('rhoméo',na=False,case=False)
gdf_obs.loc[is_rhomeo,['obse_id','obse_nom','obse_date','cd_nom']]
obs_rhomeo = gdf_obs[is_rhomeo]\
.dropna(axis=1,how='all')\
.drop(columns=['obse_habi_id'])
rhoisna = obs_rhomeo.cd_nom.isna()
obs_rhomeo = drop_cdnom_missing(obs_rhomeo)
odat = get_odat_obse(obs_rhomeo.obse_id)
OBS_RHOMEO = obs_rhomeo.merge(
format_odat(obs_rhomeo,odat),
left_on='obse_id',right_on='odat_obse_id',how='left'
).drop(columns='odat_obse_id')
OBS_RHOMEO.to_file('/home/colas/Documents/9_PROJETS/4_SICEN/RECOVERY/rhomeo_data_notin_sicen.gpkg')
RES_OBS = to_sicen(OBS_RHOMEO)
RES_OBS.to_postgis(
'saisie_observation',
con_sicen,
'saisie',
if_exists='append',
index=False,
dtype=get_columns_dtypes(
'saisie_observation',
'saisie',
con_sicen,
RES_OBS
)
)
# .to_file('/home/colas/Documents/9_PROJETS/4_SICEN/RECOVERY/rhomeo_data_notin_sicen.gpkg')
update_to_sql(
RES_OBS[['id_lot','id_origine','effectif','effectif_min']],
con_sicen,
'saisie_observation',
'saisie',
['id_lot','id_origine'],
dtype=get_columns_dtypes(
'saisie_observation',
'saisie',
con_sicen,
RES_OBS[['id_lot','id_origine','effectif','effectif_min']]
)
)
# Récupération des relevés
id_relvrho = gdf_obs[is_rhomeo].obse_relv_id.unique()
relv = list_releve(id_relv=id_relvrho)
relv_rh = relv.relv_nom
1_SICEN/correct_data_notTaxref.py
+107
View File
@@ -0,0 +1,107 @@
from pycen import con_gn,con_sicen, update_to_sql
import pandas as pd
############################
####### Araignée sp. #######
######## Fungi sp. #########
############################
dict_tax = {
'Pachygnatha sp':195648,
'Clubiona sp':190950,
'Enoplognatha sp':192179,
'Tetragnatha sp':198316,
'Linyphiidae sp':184366,
'Episinus sp':192245,
'Evarcha sp':192480,
'Thanatus sp':198365,
'Xysticus sp':199101,
'Araneidae sp':184354,
'Pardosa cf saltans':457342, # doute
'Zora sp':199158,
'Piratula sp':806893,
'Pardosa sp':195830,
'Thomisidae sp':184390,
'Gnaphosidae sp':184362,
'Araniella sp':189390,
'Pirata sp':196303,
'Trochosa sp':198691,
'Zelotes sp':199129,
'Agelena sp':188876,
'Ozyptila sp':202569,
'Metellina sp':202546,
'Hyptiotes cf paradoxus':1058, # doute
'Phrurolithus sp':196174,
'Tegenaria sp':198254,
'Singa sp':197676,
'Heliophanus sp':193148,
'Zodarion sp':199147,
'Tmarus sp':198482,
'Dysdera sp':192003,
'Liocranidae sp':184367,
'Micaria sp':194695,
'Salticidae sp':184381,
'Agroeca sp':188904,
'Lycosidae sp':184368,
'Zora cf spinimana':1753, # doute
'Marpissa sp':194474,
'Alopecosa sp':188992,
# 'Amphinema diadema':, # Contact fournisseur données : RIVOIRE Bernard
'Hyphodermella rosae':905273,
'Cartilosoma rene-hentic':905220,
# 'Phanerochaete livescens':, # Appeler CBNA / RIVOIRE Bernard
# 'Thanatephorus brevisporus':, # Contact fournisseur données : RIVOIRE Bernard
'Phlebia acerina':970280 }
sql = """
select id_obs,remarque_obs FROM saisie.saisie_observation WHERE cd_nom in ('9999077','9999078')
and remarque_obs not ilike '%%Amphinema diadema%%'
and remarque_obs not ilike '%%Phanerochaete livescens%%'
and remarque_obs not ilike '%%Thanatephorus brevisporus%%'
"""
df = pd.read_sql(sql, con_sicen)
df['cd_nom'] = (df['remarque_obs']
.str.split('|',n=1)
.str[0]
.str.replace('Nom taxon saisi :','')
.str.strip()
.replace(dict_tax)
)
sql = 'SELECT cd_nom,regne,nom_vern,phylum,famille,classe,ordre,nom_complet,nom_valide FROM taxonomie.taxref WHERE cd_nom IN {}'
tax = pd.read_sql(sql.format(tuple(df['cd_nom'].unique())), con_gn)
df = (df.merge(tax,how='left',on='cd_nom')
.drop(columns=['remarque_obs'])
.astype({'cd_nom':str}))
update_to_sql(
df,
con_sicen,
'saisie_observation',
'saisie','id_obs')
############################
######## Myotis sp. ########
############################
dict_tax = {'9999058':195005}
sql = """
select id_obs,cd_nom FROM saisie.saisie_observation WHERE cd_nom = '9999058'
"""
df = pd.read_sql(sql, con_sicen)
df.replace({'cd_nom':dict_tax},inplace=True)
sql = 'SELECT cd_nom,regne,nom_vern,phylum,famille,classe,ordre,nom_complet,nom_valide FROM taxonomie.taxref WHERE cd_nom = %i'
tax = pd.read_sql(sql%df['cd_nom'].unique()[0], con_gn)
df = (df.merge(tax,how='left',on='cd_nom')
.astype({'cd_nom':str}))
update_to_sql(
df,
con_sicen,
'saisie_observation',
'saisie','id_obs')
1_SICEN/extract_chiro.py
+39
View File
@@ -0,0 +1,39 @@
import pycen
import geopandas as gpd
sql = 'SELECT * FROM saisie.saisie_observation'
df = gpd.read_postgis(sql,pycen.con_sicen,geom_col="geometrie")
sql = 'SELECT * FROM saisie.v_synthese_invertebre'
v_inv = gpd.read_postgis(sql,pycen.con_sicen,geom_col="geom")
v_inv.date_obs = gpd.pd.to_datetime(v_inv.date_obs)
v_inv2021 = v_inv[v_inv.date_obs > '2021-01-01']
sql = 'SELECT * FROM saisie.v_synthese_vertebre'
v_vert = gpd.read_postgis(sql,pycen.con_sicen,geom_col="geom")
v_vert.date_obs = gpd.pd.to_datetime(v_vert.date_obs)
chiro = v_vert[v_vert.ordre=='Chiroptera']
dict_obs_pu = [
'{"JUTON Mathieu"}', '{"VEILLET Bruno"}',
'{"SUCHET Patrick","VEILLET Bruno"}',
'{"SUCHET Patrick"}',
'{"BIRON Nicolas"}',
'{"PASQUIER Guillaume"}',
'{"LUCAS Jérémie"}',
'{"JUTON Mathieu","PASQUIER Guillaume"}',
'{"BIRON Nicolas","BONHOMME Baptiste","GROSSI Jean-Luc","JUTON Mathieu"}',
'{"DELOCHE Denis","BEGUIN Lucile","JUTON Mathieu"}',
'{"BERENGER Myrtille"}','{"VINCENT Stéphane"}'
]
chiro[(
(chiro.diffusable=="oui") |
(chiro.observateurs.isin(dict_obs_pu)) |
chiro.rmq_localisation.str.contains('Camp de Chambaran',case=False))]
# IDEM
# chiro[(
# (chiro.diffusable=="oui") |
# (chiro.observateurs.isin(dict_obs_pu)) |
# (chiro.rmq_localisation=='Camp de Chambaran'))]
1_SICEN/extract_obs.py
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-.
from sqlalchemy import create_engine #, text
from sqlalchemy.engine import URL
import geopandas as gpd
from pycen.zh import zh
zh = zh()
from pycen import con_sicen as con
file = '/home/colas/Documents/9_PROJETS/1_ZH/SIMBY_zonage_Belledonne.txt'
if file :
lst_site = gpd.pd.read_csv(file)
lst_site = [*lst_site.site]
sql = """
SELECT * FROM zones_humides.v_zoneshumides
"""
sql = sql + " WHERE site_code IN ('%s')" % "','".join(lst_site)
VZH = gpd.read_postgis(sql,zh.con,geom_col='geom')
sql = """
SELECT * FROM saisie.saisie_observation s
WHERE ST_Intersects(s.geometrie,'SRID=2154;%s')
""" % VZH.unary_union
saisie = gpd.read_postgis(sql,con,geom_col='geometrie')
tmp = gpd.sjoin(VZH[['site_code','geom']], saisie)
ss = tmp[['site_code',
'phylum','regne','classe','ordre','famille',
'nom_valide','nom_vern','nom_complet','cd_nom']].copy()
ss.drop_duplicates(inplace=True)
ss.nom_valide = ss.nom_valide.replace([' \(#ref\)',],[''],regex=True)
ss.nom_complet = ss.nom_complet.replace([' \(#ref\)',],[''],regex=True)
ss.sort_values('nom_valide', inplace=True)
sp_enjeux = [
'Carex brunnescens (Pers.) Poir., 1813',
'Coenagrion mercuriale (Charpentier, 1840)',
'Drosera rotundifolia L., 1753',
'Juncus squarrosus L., 1753',
'Lycopodium annotinum L., 1753',
'Lycopodium clavatum L., 1753',
'Neottia cordata (L.) Rich., 1817',
'Pinguicula grandiflora subsp. rosea (Mutel) Casper, 1962',
# 'Pinguicula grandiflora Lam., 1789',
'Pinguicula grandiflora subsp. grandiflora Lam., 1789',
]
ss.nom_valide.unique()
ss[ss.nom_valide.isin(sp_enjeux)].to_excel('/home/colas/Documents/tmp/espèced_enjeux_belledonne.xlsx')
ss[ss.nom_complet.isin(sp_enjeux)].to_file('/home/colas/Documents/tmp/espèced_enjeux_belledonne.xlsx')
1_SICEN/extract_obs_bysites_toGeonature.py
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env python3
# -*- coding:UTF-8 -*-
from sqlalchemy import create_engine #, text
from sqlalchemy.engine import URL
import geopandas as gpd
import pandas as pd
from pycen import con_sicen as con, con_bdcen as con38
sql = 'SELECT * FROM sites.c_sites_zonages'
site = gpd.read_postgis(sql,con38,geom_col='geom')
site.sort_values(['code_site','surface_ha'], inplace=True)
site.reset_index(inplace=True,drop=True)
site.drop_duplicates(subset=['code_site'],keep='last', inplace=True)
cod = 'ARSE'
for cod in site.code_site:
print(cod)
sql = """
SELECT * FROM saisie.vm_synthese_observations
WHERE ST_Intersects(geom,'SRID=2154;%s')
""" % site[site.code_site==cod].geom.values[0]
obs = gpd.read_postgis(sql,con,geom_col='geom')
if obs.empty:
continue
sql = """
SELECT operation op, date_operation date_op, utilisateur user_op, id_obs FROM saisie.suivi_saisie_observation
WHERE operation = 'INSERT' AND id_obs IN (%s)
""" % ",".join(obs.id_obs.astype(str))
ins_bos = pd.read_sql_query(
sql,con
)
if ins_bos.empty: pass
else:
ins_bos['heure_op'] = ins_bos.date_op.dt.time
ins_bos.date_op = ins_bos.date_op.dt.date
ins_bos.date_op = ins_bos.date_op.astype(str)
ins_bos.heure_op = ins_bos.heure_op.astype(str)
obs = pd.merge(ins_bos,obs, on=['id_obs'], how='right')
obs.structures = [' & '.join(i) for i in obs.structures]
obs.observateurs = [' & '.join(i) for i in obs.observateurs]
obs.date_obs = obs.date_obs.astype(str)
obs.date_debut_obs = obs.date_debut_obs.astype(str)
obs.date_fin_obs = obs.date_fin_obs.astype(str)
obs.heure_obs = obs.heure_obs.astype(str)
obs.set_index('id_obs', inplace=True)
output = '/home/colas/Documents/tmp/obsbysite/%s.xlsx' % cod
with pd.ExcelWriter(output) as writer:
obs[obs.structures=='CEN Isère'].to_excel(writer,sheet_name='CEN Isère', index=True)
obs[obs.structures!='CEN Isère'].to_excel(writer,sheet_name='Partenaire', index=True)
writer.save()
1_SICEN/import_dataTOsicenIMPORT.py
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-.
from sqlalchemy import create_engine #, text
from sqlalchemy.engine import URL
from sqlalchemy.types import Date,Time,String
from shapely.geometry import Point
import pandas as pd
import datetime
from pycen import con_bdcen as con
file = '/home/colas/Documents/9_PROJETS/4_SICEN/IMPORT/Tableau Saisie_FauneFlore_AG_30_03_22.xlsx'
# file = '/home/colas/Documents/9_PROJETS/4_SICEN/LPO/cdnom_idvisionature.csv'
to_table = 'import_data_cen38'
# to_table = 'taxons_fauneisere_biolovision_taxref'
df = pd.read_excel(
file,
skiprows=[1,2,3,4]
)
if 'nom de champ' in df.columns:
del df['nom de champ']
# df = pd.read_csv(file,sep=',')
df.columns = df.columns.str.strip().str.lower()
df.dropna(subset=['date_obs'],inplace=True)
df.drop_duplicates(inplace=True)
date_default = datetime.datetime(1899, 12, 30, 0, 0)
# data_import.loc[data_import['time_start'] == date_default,'time_start'] = datetime.time(0,0)
# data_import.loc[data_import['time_stop'] == date_default,'time_stop'] = datetime.time(0,0)
# data_import.loc[data_import['time_start'] == '1899-12-30 00:00:00','time_start'] = '00:00:00'
# data_import.loc[data_import['time_stop'] == '1899-12-30 00:00:00','time_stop'] = '00:00:00'
# df.loc[df['horaire'] == date_default,'horaire'] = datetime.time(0,0)
df.date_obs = df.date_obs.dt.date
if to_table == 'import_data_cen38':
dtype = {
'date_obs':Date,
'date_debut_obs':Date,
'date_fin_obs':Date,
'heure':Time,
'strate':String,
'phenologie':String,
'cd_nom':String,
'sexe':String,
'precision':String}
else : dtype = None
df.to_sql(
name=to_table,
con = con,
schema='import',
index=False,
if_exists='replace',
method='multi',
dtype=dtype
)
sql_grant = """
GRANT ALL ON TABLE import.%s TO grp_admin
""" % to_table
with con.begin() as cnx:
cnx.execute(sql_grant)
1_SICEN/import_donneesaisieTOsicen.py
+319
View File
@@ -0,0 +1,319 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-.
from sqlalchemy import create_engine, text
from sqlalchemy.engine import URL
from datetime import datetime as dt
import pandas as pd
import geopandas as gpd
from pycen import con_sicen as con
nom_numerateur = 'GEIER'
pnom_numerator = 'Colas'
today = dt.now().date().isoformat()
nom_numerateur = nom_numerateur.upper()
pnom_numerator = pnom_numerator[0].upper() + pnom_numerator[1:]
query_get_numerator = """
SELECT * FROM md.personne
WHERE nom = '%s'
AND prenom = '%s'
AND id_structure = 1
""" % (nom_numerateur,pnom_numerator)
numerateur = pd.read_sql_query(
sql = query_get_numerator,
con = con
)
select_import = """
SELECT * FROM import.import_data_cen38;
"""
data_import = pd.read_sql_query(
sql = select_import,
con = con)
check_datageom = """
DROP VIEW IF EXISTS import.test_geom_prestation;
CREATE OR REPLACE VIEW import.test_geom_prestation as (
SELECT ROW_NUMBER() OVER (ORDER BY date_obs ASC) AS gid, localisation,
CASE WHEN latitude::numeric > 6000000
THEN ST_GeomFromText('POINT('||longitude::double precision||' '||latitude::double precision||')',2154) -- coordonnées en lambert93
WHEN latitude::numeric > 50 AND latitude::numeric < 3000000
THEN ST_transform(ST_GeomFromText('POINT('||longitude::double precision||' '||latitude::double precision||')',27572),2154) -- coordonnées en Lambert II étendu
--WHEN latitude is null THEN st_pointonsurface(b.geom) -- pas de coordonnées XY on récupère la geom du centroide du site CenRA si elle existe et si une partie du nom du site est renseignée dans la colonne LOCALISATION
ELSE ST_transform(ST_GeomFromText('POINT('||longitude::double precision||' '||latitude::double precision||')',4326),2154) -- coordonnées en WGS84
END as geometrie
FROM import.import_data_cen38
--LEFT JOIN import.sites_serena_cren_mu_2015 b ON lower(b.nom_site) like '%'||lower(trim(localisation,'s'))||'%' -- on joint la table des sites CenRA pour récupérer éventuellement le centroide d'un site si pas de XY
WHERE cd_nom is not null and (date_obs is not null or date_debut_obs is not null)) ; -- pour ignorer les lignes vides qui traineraient dans le csv
SELECT * FROM import.test_geom_prestation;
"""
df_check_datageom = gpd.read_postgis(
sql = text(check_datageom),
con = con,
geom_col='geometrie')
if df_check_datageom.geometry.is_valid.all() \
and df_check_datageom.crs.srs == 'epsg:2154':
drop_datageom = """DROP VIEW IF EXISTS import.test_geom_prestation;"""
with con.begin() as cnx:
cnx.execute(drop_datageom)
create_table_pers = """
DROP TABLE IF EXISTS md.temp_import_personne;
CREATE TABLE md.temp_import_personne AS
WITH personnel AS (
SELECT a.id_personne,a.nom,a.prenom,b.id_structure,b.nom_structure
FROM md.personne a
JOIN md.structure b USING (id_structure)
)
SELECT DISTINCT
-- personne
CASE WHEN observateur2 is null THEN (c.nom||' '||c.prenom)::text
WHEN observateur2 is not null AND observateur3 is null THEN (c.nom||' '||c.prenom)||'&'||(d.nom||' '||d.prenom)
WHEN observateur3 is not null AND observateur4 is null THEN (c.nom||' '||c.prenom)||'&'||(d.nom||' '||d.prenom)||'&'||(e.nom||' '||e.prenom)
WHEN observateur4 is not null AND observateur5 is null THEN (c.nom||' '||c.prenom)||'&'||(d.nom||' '||d.prenom)||'&'||(e.nom||' '||e.prenom)||'&'||(f.nom||' '||f.prenom)
ELSE c.nom_structure||'&'||d.nom_structure||'&'||(e.nom||' '||e.prenom)||'&'||(f.nom||' '||f.prenom)||'&'||(g.nom||' '||g.prenom)
END AS personne,
-- nom_structure
CASE WHEN observateur2 is null THEN c.nom_structure::text
WHEN observateur2 is not null AND observateur3 is null AND c.nom_structure <> d.nom_structure
THEN c.nom_structure||'&'||d.nom_structure
WHEN observateur2 is not null AND observateur3 is null AND c.nom_structure = d.nom_structure
THEN c.nom_structure
WHEN observateur3 is not null AND observateur4 is null AND c.nom_structure <> d.nom_structure AND d.nom_structure <> e.nom_structure
AND c.nom_structure <> e.nom_structure
THEN c.nom_structure||'&'||d.nom_structure||'&'||e.nom_structure
WHEN observateur3 is not null AND observateur4 is null AND c.nom_structure = d.nom_structure AND d.nom_structure <> e.nom_structure
THEN c.nom_structure||'&'||e.nom_structure
WHEN observateur3 is not null AND observateur4 is null AND c.nom_structure <> d.nom_structure AND d.nom_structure = e.nom_structure
THEN c.nom_structure||'&'||d.nom_structure
WHEN observateur4 is not null AND observateur5 is null AND c.nom_structure <> d.nom_structure
THEN c.nom_structure||'&'||d.nom_structure||'&'||e.nom_structure||'&'||f.nom_structure
ELSE c.nom_structure||'&'||d.nom_structure||'&'||e.nom_structure||'&'||f.nom_structure||'&'||g.nom_structure
END AS nom_structure,
-- observateur
CASE WHEN observateur2 is null THEN c.id_personne::text
WHEN observateur2 is not null AND observateur3 is null THEN c.id_personne||'&'||d.id_personne
WHEN observateur3 is not null AND observateur4 is null THEN c.id_personne||'&'||d.id_personne||'&'||e.id_personne
WHEN observateur4 is not null AND observateur5 is null THEN c.id_personne||'&'||d.id_personne||'&'||e.id_personne||'&'||f.id_personne
ELSE c.id_personne||'&'||d.id_personne||'&'||e.id_personne||'&'||f.id_personne||'&'||g.id_personne
END AS observateur,
structure
FROM import.import_data_cen38 a
LEFT JOIN personnel c ON (c.nom||' '||c.prenom) = trim(a.observateur1::text) AND lower(c.nom_structure) = lower(a.structure)
LEFT JOIN personnel d ON (d.nom||' '||d.prenom) = trim(a.observateur2::text) AND lower(d.nom_structure) = lower(a.structure)
LEFT JOIN personnel e ON (e.nom||' '||e.prenom) = trim(a.observateur3::text) AND lower(e.nom_structure) = lower(a.structure)
LEFT JOIN personnel f ON (f.nom||' '||f.prenom) = trim(a.observateur4::text) AND lower(f.nom_structure) = lower(a.structure)
LEFT JOIN personnel g ON (g.nom||' '||g.prenom) = trim(a.observateur5::text) AND lower(g.nom_structure) = lower(a.structure)
;
""".format(id_creator = numerateur.id_personne.item())
with con.begin() as cnx:
cnx.execute(create_table_pers)
select_pers = 'SELECT * FROM md.temp_import_personne'
df_pers = pd.read_sql_query(
sql = select_pers,
con = con)
with_dataTOinsert = """
WITH personnel AS (
SELECT a.id_personne,a.nom,a.prenom,b.id_structure,b.nom_structure
FROM md.personne a
JOIN md.structure b USING (id_structure)
),
t as (
SELECT
CASE WHEN a.heure IS NULL THEN '00:00:00'
ELSE a.heure END heure,
a.date_obs, a.date_debut_obs, a.date_fin_obs, a.duree as date_textuelle,
CASE WHEN b.regne IS NULL THEN bb.regne
ELSE b.regne END regne,
CASE WHEN b.phylum IS NULL THEN bb.phylum
ELSE b.phylum END phylum,
CASE WHEN b.classe IS NULL THEN bb.classe
ELSE b.classe END classe,
CASE WHEN b.ordre IS NULL THEN bb.ordre
ELSE b.ordre END ordre,
CASE WHEN b.famille IS NULL THEN bb.famille
ELSE b.famille END famille,
CASE WHEN b.nom_valide IS NULL THEN bb.nom_valide
ELSE b.nom_valide END nom_valide,
CASE WHEN b.nom_vern IS NULL THEN bb.nom_vern
ELSE b.nom_vern END nom_vern,
CASE WHEN b.nom_complet IS NULL THEN bb.nom_complet
ELSE b.nom_complet END nom_complet,
CASE WHEN b.cd_nom IS NULL THEN bb.cd_nom::text
ELSE b.cd_nom END cd_nom,
a.abondance_dominance as effectif_textuel, a.effectif_min, a.effectif_max,
CASE WHEN strate::text is not null THEN strate::text
ELSE age::text END AS type_effectif,
CASE WHEN phenologie is not null THEN phenologie
ELSE sexe END AS phenologie,
CASE WHEN latitude::numeric > 6000000
THEN ST_X(st_transform(ST_GeomFromText('POINT('||longitude::double precision||' '||latitude::double precision||')',2154),4326)) -- coordonnée X en lambert93
WHEN latitude::numeric > 50 AND latitude::numeric < 3000000
THEN ST_X(st_transform(ST_GeomFromText('POINT('||longitude::double precision||' '||latitude::double precision||')',27572),4326)) -- coordonnées en Lambert II étendu
-- WHEN latitude is null
-- THEN ST_X(st_transform(st_pointonsurface(i.geom),4326)) -- pas de coordonnées XY on récupère la geom du centroide du site CenRA si elle existe et si une partie du nom du site est renseignée dans la colonne LOCALISATION
ELSE longitude::double precision -- coordonnées en WGS84
END as longitude,
CASE WHEN latitude::numeric > 6000000
THEN ST_Y(st_transform(ST_GeomFromText('POINT('||longitude::double precision||' '||latitude::double precision||')',2154),4326)) -- coordonnée X en lambert93
WHEN latitude::numeric > 50 AND latitude::numeric < 3000000
THEN ST_Y(st_transform(ST_GeomFromText('POINT('||longitude::double precision||' '||latitude::double precision||')',27572),4326)) -- coordonnées en Lambert II étendu
-- WHEN latitude is null
-- THEN ST_Y(st_transform(st_pointonsurface(i.geom),4326)) -- pas de coordonnées XY on récupère la geom du centroide du site CenRA si elle existe et si une partie du nom du site est renseignée dans la colonne LOCALISATION
ELSE latitude::double precision
END as latitude,
localisation,
CASE WHEN observateur2 is null THEN c.id_personne::text
WHEN observateur2 is not null AND observateur3 is null THEN c.id_personne||'&'||d.id_personne
WHEN observateur3 is not null AND observateur4 is null THEN c.id_personne||'&'||d.id_personne||'&'||e.id_personne
WHEN observateur4 is not null AND observateur5 is null THEN c.id_personne||'&'||d.id_personne||'&'||e.id_personne||'&'||f.id_personne
ELSE c.id_personne||'&'||d.id_personne||'&'||e.id_personne||'&'||f.id_personne||'&'||g.id_personne
END AS observateur,
{id_numerateur}::integer as numerisateur, -- adapter l'identifiant du numérisateur en fonction de la personne qui réalise l'import en masse
-- cas d'une prestation on ajoute le CenRA à la structure de l'observateur '&1' sinon il faut supprimer
CASE WHEN h.id_structure <> 1 THEN (h.id_structure||'&1')::text
ELSE h.id_structure::text END as structure,
remarque_obs,
CASE WHEN lower(a.structure) like '%CEN Isère%' THEN true::boolean
ELSE false::boolean END as diffusable,
precision::saisie.enum_precision as precision,
-- CASE
-- WHEN latitude is null AND round(st_maxdistance(st_PointOnSurface(i.geom), st_convexhull(i.geom))::numeric,-2) < 101 THEN '10 à 100m'::saisie.enum_precision
-- WHEN latitude is null AND round(st_maxdistance(st_PointOnSurface(i.geom), st_convexhull(i.geom))::numeric,-2) < 501 AND round(st_maxdistance(ST_PointOnSurface(i.geom), st_convexhull(i.geom))::numeric,-2) > 100 THEN '100 à 500m'::saisie.enum_precision
-- WHEN latitude is null AND round(st_maxdistance(st_PointOnSurface(i.geom), st_convexhull(i.geom))::numeric,-2) < 1001 AND round(st_maxdistance(ST_PointOnSurface(i.geom), st_convexhull(i.geom))::numeric,-2) > 500 THEN '500 à 1000m'::saisie.enum_precision
-- WHEN latitude is null AND round(st_maxdistance(st_PointOnSurface(i.geom), st_convexhull(i.geom))::numeric,-2) > 1001 THEN '> 1000m'::saisie.enum_precision
-- ELSE precision::saisie.enum_precision
-- END as precision,
'validée'::saisie.enum_statut_validation as statut_validation,
j.id_etude,
k.id_protocole,
effectif::integer,
CASE WHEN statut_repro::text IS NOT NULL THEN (statut_repro::text)::saisie.enum_reprostatut
ELSE NULL END reprostatut,
CASE WHEN latitude::numeric > 6000000
THEN ST_GeomFromText('POINT('||longitude::double precision||' '||latitude::double precision||')',2154) -- coordonnées en lambert93
WHEN latitude::numeric > 50 AND latitude::numeric < 3000000
THEN ST_transform(ST_GeomFromText('POINT('||longitude::double precision||' '||latitude::double precision||')',27572),2154) -- coordonnées en Lambert II étendu
-- WHEN latitude is null
-- THEN st_pointonsurface(i.geom) -- pas de coordonnées XY on récupère la geom du centroide du site CenRA si elle existe et si une partie du nom du site est renseignée dans la colonne LOCALISATION
ELSE ST_transform(ST_GeomFromText('POINT('||longitude::double precision||' '||latitude::double precision||')',4326),2154) -- coordonnées en WGS84
END as geometrie,
code_releve,
CASE WHEN obs_null::text ilike 'oui' THEN true::boolean
ELSE false::boolean END as obs_null
FROM import.import_data_cen38 a
LEFT JOIN inpn.taxref b ON b.cd_nom::text = a.cd_nom::text
LEFT JOIN inpn.taxref bb ON bb.nom_complet::text = a.nom_complet::text
LEFT JOIN personnel c ON (c.nom||' '||c.prenom) = trim(a.observateur1::text) AND lower(c.nom_structure) = lower(a.structure)
LEFT JOIN personnel d ON (d.nom||' '||d.prenom) = trim(a.observateur2::text) AND lower(d.nom_structure) = lower(a.structure)
LEFT JOIN personnel e ON (e.nom||' '||e.prenom) = trim(a.observateur3::text) AND lower(e.nom_structure) = lower(a.structure)
LEFT JOIN personnel f ON (f.nom||' '||f.prenom) = trim(a.observateur4::text) AND lower(f.nom_structure) = lower(a.structure)
LEFT JOIN personnel g ON (g.nom||' '||g.prenom) = trim(a.observateur5::text) AND lower(g.nom_structure) = lower(a.structure)
JOIN md.structure h ON lower(h.nom_structure) = lower(replace(replace(a.structure,' & ','&'),'CEN38&',''))
LEFT JOIN md.etude j on a.id_etude = j.nom_etude
LEFT JOIN md.protocole k USING (id_protocole)
--LEFT JOIN import.sites_serena_cren_mu_2015 i ON lower(i.nom_site) like '%'||lower(trim(localisation,'s'))||'%' -- on joint la table des sites CenRA pour récupérer éventuellement le centroide d'un site si pas de XY
--JOIN md.etude i ON i.nom_etude = a.id_etude -- à utiliser si les champs id_etude et id_protocole contiennent le libelle plutôt que l'identifiant, modifier aussi au-dessus id_etude en i.id_etude, id_protocole en j.id_protocole
--JOIN md.protocole j ON j.libelle = a.id_protocole
)
""".format(id_numerateur=numerateur.id_personne.item())
select_dataTOinsert = """
SELECT DISTINCT
heure::TIME WITHOUT TIME ZONE as heure_obs, date_obs, date_debut_obs, date_fin_obs, date_textuelle,
regne, nom_vern, nom_complet, cd_nom, effectif_textuel,
effectif_min, effectif_max, type_effectif, phenologie,
longitude, latitude,
localisation,
observateur, numerisateur,
structure, remarque_obs,
c.code_insee,
diffusable, "precision",
statut_validation,
id_etude, id_protocole,
effectif, reprostatut,
t.geometrie,
code_releve id_waypoint,
obs_null
FROM t, ign_bd_topo.commune c
WHERE ST_intersects(t.geometrie ,c.geometrie)
--ORDER BY nom_complet, date_obs, longitude, latitude
;
"""
df_data_insert = gpd.read_postgis(
sql = text(with_dataTOinsert+select_dataTOinsert),
con = con,
geom_col='geometrie')
# pd.read_sql_query(
# sql = text(with_dataTOinsert+select_dataTOinsert),
# con = con,)
if df_data_insert.shape[0] == data_import.shape[0]:
insert_data = """
INSERT INTO saisie.saisie_observation(
heure_obs, date_obs, date_debut_obs, date_fin_obs, date_textuelle, regne, nom_vern, nom_complet, cd_nom, effectif_textuel,
effectif_min, effectif_max, type_effectif, phenologie, longitude, latitude, localisation, observateur, numerisateur, structure, remarque_obs, code_insee, diffusable, "precision",
statut_validation, id_etude, id_protocole, effectif, reprostatut, geometrie, id_waypoint, obs_null)
"""
with con.begin() as cnx:
cnx.execute(
text(with_dataTOinsert+
insert_data+
select_dataTOinsert)
)
update_structure_name = """
UPDATE saisie.suivi_saisie_observation a
SET utilisateur = REPLACE(utilisateur, 'inconnu', '{email_numerator}')
WHERE date_operation > '{today}'
;
""".format(
email_numerator=numerateur.email.item(),
today = today
)
drop_temp_table = """
DROP TABLE IF EXISTS md.temp_import_personne;
"""
with con.begin() as cnx:
cnx.execute(update_structure_name)
cnx.execute(drop_temp_table)
else :
print(data_import[~data_import.index.isin(df_data_insert.index)])
data_import[~data_import.index.isin(data_import.drop_duplicates().index)]
check_datageom = """
DROP VIEW IF EXISTS import.test_geom_prestation;
"""
with con.begin() as cnx:
cnx.execute(check_datageom)
1_SICEN/import_fauneisereTOsicen (biolovision).py
+457
View File
@@ -0,0 +1,457 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-.
from sqlalchemy import create_engine, text
from sqlalchemy.engine import URL
from datetime import datetime as dt
import pandas as pd
import geopandas as gpd
from pycen import con_sicen as con
nom_numerateur = 'GEIER'
pnom_numerator = 'Colas'
today = dt.now().date().isoformat()
nom_numerateur = nom_numerateur.upper()
pnom_numerator = pnom_numerator[0].upper() + pnom_numerator[1:]
query_get_numerator = """
SELECT * FROM md.personne
WHERE nom = '%s'
AND prenom = '%s'
AND id_structure = 1
""" % (nom_numerateur,pnom_numerator)
numerateur = pd.read_sql_query(
sql = query_get_numerator,
con = con
)
select_import = """
SELECT * FROM import.import_data_fauneisere_biolovision;
"""
data_import = pd.read_sql_query(
sql = select_import,
con = con)
verif_taxref = '''
SELECT CASE WHEN c.taxref_cdnom IS NULL THEN bb.cd_nom
ELSE c.taxref_cdnom END cdnom_taxref,
CASE WHEN c.cd_nom IS NULL THEN cc.cd_nom
ELSE c.cd_nom END cd_nom,
CASE WHEN c.cd_ref IS NULL THEN cc.cd_ref
ELSE c.cd_ref END cd_ref,
a.ref, a."id espèce biolovision", a."nom espèce", a."nom latin",
a.date, a.jour, a.mois, a.année, a."jour de l'année", a.pentade, a."décade",
a."numéro de la semaine", a."heure début", a."heure de début",
a."minute de début", a."heure fin", a."heure de fin", a."minute de fin", a.horaire,
--a.id_place, a.commune, a.municipality, a.county, a.country, "code insee", a.altitude,
a."lat (wgs84)", a."lon (wgs84)", a."type de localisation",
a.estimation, a.nombre, a."détails", a."code atlas", a.comportement,
a."prénom", a.nom, a."abréviation personne morale",
--a.search_export_species_sheet_protection_nat, a.search_export_entity_full_name, a.search_export_entity_tra_full_name,
a.search_export_species_sheet_dir_oiseaux, a.search_export_species_sheet_dir_habitat,
a.search_export_species_sheet_uicn, a.search_export_species_sheet_lr_nat, a.search_export_species_sheet_znieff,
a.search_export_species_sheet_id_fnat,
--a."contient des détails mortalité", a.protocol,
a.remarque, a."remarque privée",a."date d'insertion", a."date de dernière modification"
FROM import.import_data_fauneisere_biolovision a
LEFT JOIN (
SELECT DISTINCT b.taxref_cdnom, b.id_visio, t.*
FROM ref_lpo.taxons_fauneisere_biolovision_taxref b
JOIN inpn.taxref t ON t.cd_nom::integer = b.taxref_cdnom) c
ON a."id espèce biolovision" = c.id_visio
LEFT JOIN (inpn.taxons_isere_absents_taxref bb
JOIN inpn.taxref cc ON cc.cd_nom::integer = bb.cd_nom)
ON a."id espèce biolovision" = bb.id_taxon
;
'''
df_verif = pd.read_sql_query(
sql = verif_taxref,
con = con)
if df_verif.shape[0] != data_import.shape[0]:
print('shape data_import : %s') % data_import.shape[0]
print('shape taxref_verif : %s') % df_verif.shape[0]
raise Exception('ERROR : correspondance with TAXREF invalid !')
# df_verif[df_verif.cd_nom.isna()][["id espèce biolovision","nom espèce","nom latin"]].drop_duplicates()
# data_import[~data_import.id_species.isin(df_verif.id_species)] \
# [['name_species','latin_species','id_species']].drop_duplicates()
update_structure_name = """
UPDATE import.import_data_fauneisere_biolovision a
SET "abréviation personne morale" = REPLACE("abréviation personne morale", 'CEN38', 'CEN Isère');
"""
with con.begin() as cnx:
cnx.execute(update_structure_name)
create_table_pers = """
DROP TABLE IF EXISTS md.temp_import_personne;
CREATE TABLE md.temp_import_personne AS
WITH nom_complet_identifies AS
(SELECT DISTINCT
ltrim(unnest(string_to_array(a."prénom",'&'))) as prenom,
ltrim(unnest(CASE WHEN "abréviation personne morale" IN ('CEN Isère','-') THEN string_to_array(''|| upper(split_part(a.nom,' /',1)),'&')
ELSE string_to_array(upper(split_part(a.nom,' (',1)),'&') END)) as nom,
CASE WHEN "abréviation personne morale" ILIKE '-' THEN 'CEN Isère'
ELSE "abréviation personne morale" END as structure,
a.nom || ' ' || a."prénom" as full_name
FROM import.import_data_fauneisere_biolovision a
ORDER BY full_name),
personne_sicen AS
(SELECT a.id_personne, prenom, nom, nom || ' ' || prenom as nom_complet, role, specialite,
a.id_structure, b.nom_structure
FROM md.personne a
JOIN md.structure b ON a.id_structure = b.id_structure
--WHERE personne.id_personne BETWEEN 100000 AND 999998
)
SELECT DISTINCT
CASE WHEN b.id_personne IS NULL THEN nextval('md.personne_id_personne_hors_cen38_seq'::regclass)
ELSE b.id_personne END as id_personne,
b.id_personne id_personne_exist,
'Observateur Faune-Isère' as remarque,
a.prenom as prenom,
a.nom as nom,
a.full_name as nom_complet,
b.nom_structure,
a.structure,
CASE WHEN a.structure ILIKE 'CEN Isère' THEN 1
ELSE 1 END AS id_structure
FROM nom_complet_identifies a
LEFT JOIN personne_sicen b ON lower(a.nom || ' ' || a.prenom) = lower(b.nom_complet) AND b.nom_structure = a.structure
;
"""
with con.begin() as cnx:
cnx.execute(create_table_pers)
select_pers = 'SELECT * FROM md.temp_import_personne'
df_pers = pd.read_sql_query(
sql = select_pers,
con = con)
select_persTOinsert = """
SELECT DISTINCT
a.id_personne,
a.remarque,
a.prenom as prenom,
a.nom as nom, --full_name
NULL as email,
'observ'::md.enum_role,
NULL::md.enum_specialite as specialite,
0 as mot_de_passe,
{id_creator} as createur,
'Mme / M.'::md.enum_titre as titre,
now()::date as date_maj,
a.id_structure
FROM md.temp_import_personne a
WHERE NOT EXISTS (SELECT DISTINCT id_personne FROM md.personne)
;
""".format(id_creator = numerateur.id_personne.item())
df_pers_insert = pd.read_sql_query(
sql = select_persTOinsert,
con = con)
if not df_pers_insert.empty:
raise Exception("DEV en cours ...")
insert_pers = """
INSERT INTO md.personne(
id_personne,
remarque,
prenom,
nom,
email,
role,
specialite,
mot_de_passe,
createur,
titre,
date_maj,
id_structure)
""" + select_persTOinsert
with con.begin() as cnx:
cnx.execute(insert_pers)
else:
pass
select_pers = """
WITH observateurs_multiples AS
(SELECT DISTINCT
string_agg(a.id_personne::text,'&') as id_personne,
--a.remarque,
--a.prenom as prenom,
--a.nom as nom,
a.nom_complet,
a.id_structure,
s.nom_structure
FROM md.temp_import_personne a
LEFT JOIN md.structure s ON a.id_structure = s.id_structure
GROUP BY a.nom_complet, a.id_structure,s.nom_structure)
"""
select_dataTOinsert = """
SELECT DISTINCT
--a."id espèce biolovision",
--a.name_species,
--a.latin_species,
--a.protegee,
--a.embr,
--a.classe1,
--a.classe2,
a.date::date,
CASE WHEN a.horaire <> '00:00:00' THEN a.horaire::time
ELSE NULL END as heure_obs,
--a.date_year,
--a.time_start,
--a.time_stop,
CASE WHEN t.regne IS NULL THEN tt.regne
ELSE t.regne END regne,
CASE WHEN t.phylum IS NULL THEN tt.phylum
ELSE t.phylum END phylum,
CASE WHEN t.classe IS NULL THEN tt.classe
ELSE t.classe END classe,
CASE WHEN t.ordre IS NULL THEN tt.ordre
ELSE t.ordre END ordre,
CASE WHEN t.famille IS NULL THEN tt.famille
ELSE t.famille END famille,
CASE WHEN t.nom_valide IS NULL THEN tt.nom_valide
ELSE t.nom_valide END nom_valide,
CASE WHEN t.nom_vern IS NULL THEN tt.nom_vern
ELSE t.nom_vern END nom_vern,
CASE WHEN t.nom_complet IS NULL THEN tt.nom_complet
ELSE t.nom_complet END nom_complet,
CASE WHEN t.cd_nom IS NULL THEN tt.cd_nom
ELSE t.cd_nom END cd_nom,
--t.cd_nom cd_nom_2,
--b.taxref_cdnom,
CASE WHEN a.estimation IS NULL AND (a.nombre IS NOT NULL OR a.nombre > 0) THEN a.nombre
WHEN a.estimation LIKE 'x' THEN '1'
ELSE NULL END AS effectif,
CASE WHEN a.estimation LIKE '>' AND (a.nombre IS NOT NULL OR a.nombre > 0) THEN a.nombre
WHEN a.estimation LIKE '~' AND (a.nombre IS NOT NULL OR a.nombre > 0) THEN a.nombre
ELSE NULL END AS effectif_min,
CASE WHEN a.estimation LIKE '~' AND (a.nombre IS NOT NULL OR a.nombre > 0) THEN (a.nombre + 10)
ELSE NULL END AS effectif_max,
a."lon (wgs84)" as longitude,
a."lat (wgs84)" as latitude,
a.commune as localisation,
o.id_personne as observateur,
/*a."prénom",
a.nom,
a.entity_short_name,
a.entity_full_name,
a.tra_surname,
a.tra_name,
a.tra_full_name,*/
{id_numerateur} as numerisateur,
o.id_structure,
concat_ws(' | ',
CASE WHEN a."détails" IS NOT NULL THEN 'Détails : '|| a."détails" ELSE NULL END,
CASE WHEN a."contient des détails mortalité" ILIKE 'oui' THEN 'Mortalité' ELSE NULL END,
CASE WHEN a.comportement IS NOT NULL THEN 'Comportement : '|| a.comportement ELSE NULL END,
CASE WHEN a."code atlas" IS NOT NULL THEN 'Code atlas : '|| c.categorie || CONCAT(' (',c.libelle,')') ELSE NULL END,
CASE WHEN a.remarque IS NOT NULL THEN 'Commentaires : '|| a.remarque ELSE NULL END
) as remarque_obs,
--a.id_place,
--a.municipality,
--a.county,
--a.country,
"code insee" as code_insee,
TRUE as diffusable,
--CASE WHEN a.hidden ILIKE 'oui' THEN TRUE ELSE FALSE END as diffusable,
CASE WHEN lower(a."type de localisation") ILIKE 'lieu-dit' THEN 'lieu-dit'
WHEN lower(a."type de localisation") ILIKE 'localisation précise' THEN '0 à 10m'
ELSE NULL END::saisie.enum_precision AS "precision",
--a.grid_name,
--a.estimation,
'validée'::saisie.enum_statut_validation as statut_validation,
99 as id_etude, -- Echange de données (partenaires)
3 as id_protocole, -- Observation aléatoire
a.altitude as elevation,
--a.hidden,
--a.admin_hidden,
--a.committee_chn,
--a.committee_chr,
--a.second_hand,
--a.private_comment,
--a.insert_date,
--a.update_date,
--a.project_code,
ST_Transform(st_setsrid(st_makepoint("lon (wgs84)", "lat (wgs84)"),4326),2154)::geometry(POINT,2154) AS geometrie,
CASE WHEN (a.nombre IS NULL OR a.nombre = 0) THEN TRUE ELSE FALSE END AS obs_null,
8 as id_lot, -- Code du lot de données : Import Faune-Isère (exporté par LPO)
a.ref as id_origine
FROM import.import_data_fauneisere_biolovision a
LEFT JOIN (
SELECT DISTINCT b.taxref_cdnom, b.id_visio, t.*
FROM ref_lpo.taxons_fauneisere_biolovision_taxref b
JOIN inpn.taxref t ON t.cd_nom::integer = b.taxref_cdnom) t
ON a."id espèce biolovision" = t.id_visio
LEFT JOIN (inpn.taxons_isere_absents_taxref bb
JOIN inpn.taxref tt ON tt.cd_nom::integer = bb.cd_nom)
ON a."id espèce biolovision" = bb.id_taxon
LEFT JOIN ref_lpo.tr_code_atlas_lpo c ON a."code atlas" = c.code_atlas
JOIN observateurs_multiples o ON lower(a.nom || ' ' || a."prénom") = lower(o.nom_complet)
--LEFT JOIN saisie.saisie_observation s
--ON ((a.ref::text = s.id_origine) AND (id_lot IN (7.8)))
--AND s.id_origine IS NULL
;
""".format(id_numerateur=numerateur.id_personne.item())
df_data_insert = gpd.read_postgis(
sql = text(select_pers+select_dataTOinsert),
con = con,
geom_col='geometrie')
# df_data_insert[df_data_insert.id_origine.duplicated(keep=False)]
# df_data_insert[df_data_insert.id_origine.duplicated(keep=False)].cd_nom.unique()
# df_data_insert[df_data_insert.cd_nom.isna()]
if df_data_insert.shape[0] == data_import.shape[0]:
insert_data = """
INSERT INTO saisie.saisie_observation(
--id_obs,
date_obs,
--date_debut_obs,
--date_fin_obs,
heure_obs,
--date_textuelle,
regne,
phylum,
classe,
ordre,
famille,
nom_valide,
nom_vern,
nom_complet,
cd_nom,
--effectif_textuel,
effectif,
effectif_min,
effectif_max,
--type_effectif,
--phenologie,
--id_waypoint,
longitude,
latitude,
localisation,
observateur,
numerisateur,
--validateur,
structure,
remarque_obs,
code_insee,
--id_lieu_dit,
diffusable,
"precision",
statut_validation,
id_etude,
id_protocole,
--url_photo,
--commentaire_photo,
--decision_validation,
--determination,
elevation,
geometrie,
--qualification,
--reprostatut,
obs_null,
--uuid,
id_lot,
id_origine)
"""
with con.begin() as cnx:
cnx.execute(
text(select_pers+
insert_data+
select_dataTOinsert)
)
update_structure_name = """
UPDATE saisie.suivi_saisie_observation a
SET utilisateur = REPLACE(utilisateur, 'inconnu', '{email_numerator}')
WHERE date_operation > '{today}'
;
""".format(
email_numerator=numerateur.email.item(),
today = today
)
drop_temp_table = """
DROP TABLE IF EXISTS md.temp_import_personne;
"""
with con.begin() as cnx:
cnx.execute(update_structure_name)
cnx.execute(drop_temp_table)
{
'ref': 'ref',
"id espèce biolovision": 'id_species',
"lat (wgs84)": 'coord_lat',
"lon (wgs84)": 'coord_lon',
"nom espèce": 'name_species',
"nom latin": 'latin_species',
# 'abréviation de la personne morale du transmetteur',
"prénom":'surname',
'nom':'name',
'abréviation personne morale':'search_export_entity_short_name',
# 'personne morale',
'nombre': 'total_count',
'estimation': 'estimation_code',
'jour':'date_day',
'mois':'date_month',
'année':'date_year',
"jour de l'année":'date_jday',
'pentade':'date_pentade',
'décade':'date_decade',
"numéro de la semaine":'date_week',
"heure fin":'time_stop',
"heure de fin":'time_stop_hour',
"minute de fin":'time_stop_min',
"heure début":'time_start',
"heure de début":'time_start_hour',
"minute de début":'time_start_min',
# 'id universel transmetteur',
# 'id transmetteur',
# 'e-mail',
# 'personne morale du transmetteur',
# 'id universel observateur',
# 'nom transmetteur',
# 'prénom transmetteur',
# 'transmetteur anonyme',
# 'anonyme',
# 'liste complète ?',
# 'protégée',
# 'vérification',
# 'donnée de seconde main',
# 'commentaire de la liste',
# 'ordre systématique',
"détails": 'detail',
'horaire':'timing',
"type de localisation": 'precision',
'comportement': 'behaviour',
'remarque': 'comment',
"remarque privée": 'private_comment',
# 'protection nationale',
"date d'insertion": 'insert_date',
"date de dernière modification": 'update_date',
"code atlas":'atlas_code',
"code insee": 'insee',
"altitude": 'altitude',
"commune": 'place',
"contient des détails mortalité":'has_death_info',
# 'id liste'
}
1_SICEN/import_fauneisereTOsicen.py
+391
View File
@@ -0,0 +1,391 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-.
from sqlalchemy import create_engine, text
from sqlalchemy.engine import URL
from datetime import datetime as dt
import pandas as pd
import geopandas as gpd
from pycen import con_sicen as con
nom_numerateur = 'GEIER'
pnom_numerator = 'Colas'
today = dt.now().date().isoformat()
nom_numerateur = nom_numerateur.upper()
pnom_numerator = pnom_numerator[0].upper() + pnom_numerator[1:]
query_get_numerator = """
SELECT * FROM md.personne
WHERE nom = '%s'
AND prenom = '%s'
AND id_structure = 1
""" % (nom_numerateur,pnom_numerator)
numerateur = pd.read_sql_query(
sql = query_get_numerator,
con = con
)
create_view='''
DROP VIEW import.test_import_faune_isere;
# CREATE OR REPLACE VIEW import.test_import_faune_isere AS
# SELECT DISTINCT
# *,
# st_setsrid(st_makepoint(coord_lon_l93, coord_lat_l93),2154)::geometry(POINT,2154) AS geom
# FROM import.import_data_fauneisere;
'''
with con.begin() as cnx:
cnx.execute(create_view)
select_import = """
SELECT * FROM import.import_data_fauneisere;
"""
data_import = pd.read_sql_query(
sql = select_import,
con = con)
verif_taxref = '''
SELECT CASE WHEN b.cdnom_taxref IS NULL THEN bb.cd_nom
ELSE b.cdnom_taxref END cdnom_taxref,
b.cdnom_taxref,
CASE WHEN c.cd_nom IS NULL THEN cc.cd_nom
ELSE c.cd_nom END cd_nom,
--c.cd_nom,
CASE WHEN c.cd_ref IS NULL THEN cc.cd_ref
ELSE c.cd_ref END cd_ref,
--c.cd_ref,
a.id_sighting, a.id_species, a.name_species, a.id_observer, a.latin_species,
a.taxonomy_name, a.family_name, a.sys_order, a.date, a.date_day, a.date_month, a.date_year,
a.date_jday, a.date_pentade, a.date_decade, a.date_week, a.time_start, a.time_start_hour, a.time_start_min, a.time_stop, a.time_stop_hour, a.time_stop_min,
a.full_form, a.timing, a.id_place, a.place, a.municipality, a.county, a.country, a.insee, a.coord_lon_liie, a.coord_lat_liie, a.coord_lon_l93, a.coord_lat_l93,
a.coord_lat, a.coord_lon, a.coord_lat_dms, a.coord_lon_dms, a.coord_f, a.coord_e, a.coord_n, a."precision", a.altitude, a.grid_name,
a.estimation_code, a.total_count,
a.detail, a.atlas_code,
a.hidden, a.admin_hidden, a.behaviour,
--a.committee_chn, a.committee_chr,
a.search_export_species_sheet_dir_oiseaux, a.search_export_species_sheet_protection_nat, a.search_export_species_sheet_dir_habitat,
a.search_export_species_sheet_uicn, a.search_export_species_sheet_lr_nat, a.search_export_species_sheet_znieff, a.search_export_species_sheet_id_fnat,
a.surname, a.name, a.search_export_entity_short_name, a.search_export_entity_full_name, a.tra_surname, a.tra_name, a.id_form,
a.search_export_entity_tra_full_name, a.second_hand, a.comment, a.private_comment, a.daily_text_comment_rem, a.insert_date, a.update_date, a.protocol, a.has_death_info
FROM import.import_data_fauneisere a
LEFT JOIN (ref_lpo.taxons_faune_isere_taxref b
JOIN inpn.taxref c ON c.cd_nom::integer = b.cdnom_taxref)
ON a.latin_species = b.latin_species
LEFT JOIN (inpn.taxons_isere_absents_taxref bb
JOIN inpn.taxref cc ON cc.cd_nom::integer = bb.cd_nom)
ON a.id_species = bb.id_taxon
;
'''
df_verif = pd.read_sql_query(
sql = verif_taxref,
con = con)
data_import[~data_import.id_species.isin(df_verif.id_species)] \
[['name_species','latin_species','id_species']].drop_duplicates()
update_structure_name = """
UPDATE import.import_data_fauneisere a
SET search_export_entity_short_name = REPLACE(search_export_entity_short_name, 'CEN38', 'CEN Isère');
"""
with con.begin() as cnx:
cnx.execute(update_structure_name)
create_table_pers = """
DROP TABLE IF EXISTS md.temp_import_personne;
CREATE TABLE md.temp_import_personne AS
WITH nom_complet_identifies AS
(SELECT DISTINCT
ltrim(unnest(string_to_array(a.surname,'&'))) as prenom,
ltrim(unnest(CASE WHEN search_export_entity_short_name IN ('CEN Isère','-') THEN string_to_array(''|| upper(split_part(a.name,' /',1)),'&')
ELSE string_to_array(upper(split_part(a.name,' (',1)),'&') END)) as nom,
CASE WHEN search_export_entity_short_name ILIKE '-' THEN 'CEN Isère'
ELSE search_export_entity_short_name END as structure,
a.name || ' ' || a.surname as full_name
FROM import.import_data_fauneisere a
ORDER BY full_name),
personne_sicen AS
(SELECT a.id_personne, prenom, nom, nom || ' ' || prenom as nom_complet, role, specialite,
a.id_structure, b.nom_structure
FROM md.personne a
JOIN md.structure b ON a.id_structure = b.id_structure
--WHERE personne.id_personne BETWEEN 100000 AND 999998
)
SELECT DISTINCT
CASE WHEN b.id_personne IS NULL THEN nextval('md.personne_id_personne_hors_cen38_seq'::regclass)
ELSE b.id_personne END as id_personne,
b.id_personne id_personne_exist,
'Observateur Faune-Isère' as remarque,
a.prenom as prenom,
a.nom as nom,
a.full_name as nom_complet,
b.nom_structure,
a.structure,
CASE WHEN a.structure ILIKE 'CEN Isère' THEN 1
ELSE 1 END AS id_structure
FROM nom_complet_identifies a
LEFT JOIN personne_sicen b ON lower(a.nom || ' ' || a.prenom) = lower(b.nom_complet) AND b.nom_structure = a.structure
;
"""
with con.begin() as cnx:
cnx.execute(create_table_pers)
select_persTOinsert = """
SELECT DISTINCT
a.id_personne,
a.remarque,
a.prenom as prenom,
a.nom as nom, --full_name
NULL as email,
'observ'::md.enum_role,
NULL::md.enum_specialite as specialite,
0 as mot_de_passe,
{id_creator} as createur,
'Mme / M.'::md.enum_titre as titre,
now()::date as date_maj,
a.id_structure
FROM md.temp_import_personne a
WHERE NOT EXISTS (SELECT DISTINCT id_personne FROM md.personne)
;
""".format(id_creator = numerateur.id_personne.item())
df_pers_insert = pd.read_sql_query(
sql = select_persTOinsert,
con = con)
if not df_pers_insert.empty:
raise Exception("DEV en cours ...")
insert_pers = """
INSERT INTO md.personne(
id_personne,
remarque,
prenom,
nom,
email,
role,
specialite,
mot_de_passe,
createur,
titre,
date_maj,
id_structure)
""" + select_persTOinsert
with con.begin() as cnx:
cnx.execute(insert_pers)
else:
pass
select_pers = """
WITH observateurs_multiples AS
(SELECT DISTINCT
string_agg(a.id_personne::text,'&') as id_personne,
--a.remarque,
--a.prenom as prenom,
--a.nom as nom,
a.nom_complet,
a.id_structure,
s.nom_structure
FROM md.temp_import_personne a
LEFT JOIN md.structure s ON a.id_structure = s.id_structure
GROUP BY a.nom_complet, a.id_structure,s.nom_structure)
"""
select_dataTOinsert = """
SELECT DISTINCT
--a.id_species,
--a.name_species,
--a.latin_species,
--a.protegee,
--a.embr,
--a.classe1,
--a.classe2,
a.date::date,
CASE WHEN a.timing <> '00:00:00' THEN a.timing::time
ELSE NULL END as heure_obs,
--a.date_year,
a.time_start,
a.time_stop,
CASE WHEN t.regne IS NULL THEN tt.regne
ELSE t.regne END regne,
CASE WHEN t.phylum IS NULL THEN tt.phylum
ELSE t.phylum END phylum,
CASE WHEN t.classe IS NULL THEN tt.classe
ELSE t.classe END classe,
CASE WHEN t.ordre IS NULL THEN tt.ordre
ELSE t.ordre END ordre,
CASE WHEN t.famille IS NULL THEN tt.famille
ELSE t.famille END famille,
CASE WHEN t.nom_valide IS NULL THEN tt.nom_valide
ELSE t.nom_valide END nom_valide,
CASE WHEN t.nom_vern IS NULL THEN tt.nom_vern
ELSE t.nom_vern END nom_vern,
CASE WHEN t.nom_complet IS NULL THEN tt.nom_complet
ELSE t.nom_complet END nom_complet,
CASE WHEN t.cd_nom IS NULL THEN tt.cd_nom
ELSE t.cd_nom END cd_nom,
--t.cd_nom cd_nom_2,
--b.cdnom_taxref,
CASE WHEN a.estimation_code IS NULL AND (a.total_count IS NOT NULL OR a.total_count > 0) THEN a.total_count
WHEN a.estimation_code LIKE 'x' THEN '1'
ELSE NULL END AS effectif,
CASE WHEN a.estimation_code LIKE '>' AND (a.total_count IS NOT NULL OR a.total_count > 0) THEN a.total_count
WHEN a.estimation_code LIKE '~' AND (a.total_count IS NOT NULL OR a.total_count > 0) THEN a.total_count
ELSE NULL END AS effectif_min,
CASE WHEN a.estimation_code LIKE '~' AND (a.total_count IS NOT NULL OR a.total_count > 0) THEN (a.total_count + 10)
ELSE NULL END AS effectif_max,
a.coord_lon_l93 as longitude,
a.coord_lat_l93 as latitude,
a.place as localisation,
o.id_personne as observateur,
/*a.surname,
a.name,
a.entity_short_name,
a.entity_full_name,
a.tra_surname,
a.tra_name,
a.tra_full_name,*/
{id_numerateur} as numerisateur,
o.id_structure,
concat_ws(' | ',
CASE WHEN a.detail IS NOT NULL THEN 'Détails : '|| a.detail ELSE NULL END,
--CASE WHEN a.has_death_info ILIKE 'oui' THEN 'Mortalité' ELSE NULL END,
CASE WHEN a.behaviour IS NOT NULL THEN 'Comportement : '|| a.behaviour ELSE NULL END,
CASE WHEN a.atlas_code IS NOT NULL THEN 'Code atlas : '|| a.atlas_code ELSE NULL END,
CASE WHEN a.comment IS NOT NULL THEN 'Commentaires : '|| a.comment ELSE NULL END
) as remarque_obs,
--a.id_place,
--a.municipality,
--a.county,
--a.country,
a.insee as code_insee,
TRUE as diffusable,
--CASE WHEN a.hidden ILIKE 'oui' THEN TRUE ELSE FALSE END as diffusable,
CASE WHEN lower(a."precision") ILIKE 'lieu-dit' THEN 'lieu-dit'
WHEN lower(a."precision") ILIKE 'localisation précise' THEN '0 à 10m'
ELSE NULL END::saisie.enum_precision AS "precision",
--a.grid_name,
--a.estimation_code,
'validée'::saisie.enum_statut_validation as statut_validation,
99 as id_etude, -- Echange de données (partenaires)
3 as id_protocole, -- Observation aléatoire
a.altitude as elevation,
--a.hidden,
--a.admin_hidden,
--a.committee_chn,
--a.committee_chr,
--a.second_hand,
--a.private_comment,
--a.insert_date,
--a.update_date,
--a.project_code,
st_setsrid(st_makepoint(coord_lon_l93, coord_lat_l93),2154)::geometry(POINT,2154) AS geometrie,
CASE WHEN (a.total_count IS NULL OR a.total_count = 0) THEN TRUE ELSE FALSE END AS obs_null,
8 as id_lot, -- Code du lot de données : Import Faune-Isère (exporté par LPO)
a.id_sighting as id_origine
FROM import.import_data_fauneisere a
LEFT JOIN (ref_lpo.taxons_faune_isere_taxref b
JOIN inpn.taxref t ON t.cd_nom::integer = b.cdnom_taxref)
ON a.latin_species = b.latin_species
LEFT JOIN (inpn.taxons_isere_absents_taxref bb
JOIN inpn.taxref tt ON tt.cd_nom::integer = bb.cd_nom)
ON a.id_species = bb.id_taxon
JOIN observateurs_multiples o ON lower(a.name || ' ' || a.surname) = lower(o.nom_complet)
--LEFT JOIN saisie.saisie_observation s
--ON ((a.id_sighting::text = s.id_origine) AND (id_lot IN (7.8)))
--AND s.id_origine IS NULL
;
""".format(id_numerateur=numerateur.id_personne.item())
df_data_insert = gpd.read_postgis(
sql = text(select_pers+select_dataTOinsert),
con = con,
geom_col='geometrie')
if df_data_insert.shape[0] == data_import.shape[0]:
insert_data = """
INSERT INTO saisie.saisie_observation(
--id_obs,
date_obs,
--date_debut_obs,
--date_fin_obs,
heure_obs,
--date_textuelle,
regne,
phylum,
classe,
ordre,
famille,
nom_valide,
nom_vern,
nom_complet,
cd_nom,
--effectif_textuel,
effectif,
effectif_min,
effectif_max,
--type_effectif,
--phenologie,
--id_waypoint,
longitude,
latitude,
localisation,
observateur,
numerisateur,
--validateur,
structure,
remarque_obs,
code_insee,
--id_lieu_dit,
diffusable,
"precision",
statut_validation,
id_etude,
id_protocole,
--url_photo,
--commentaire_photo,
--decision_validation,
--determination,
elevation,
geometrie,
--qualification,
--reprostatut,
obs_null,
--uuid,
id_lot,
id_origine)
"""
with con.begin() as cnx:
cnx.execute(
text(select_pers+
insert_data+
select_dataTOinsert)
)
update_structure_name = """
UPDATE saisie.suivi_saisie_observation a
SET utilisateur = REPLACE(utilisateur, 'inconnu', '{email_numerator}')
WHERE date_operation > '{today}'
;
""".format(
email_numerator=numerateur.email.item(),
today = today
)
drop_temp_table = """
DROP TABLE IF EXISTS md.temp_import_personne;
"""
with con.begin() as cnx:
cnx.execute(update_structure_name)
cnx.execute(drop_temp_table)
1_SICEN/import_serena_to_sicen.py
+484
View File
@@ -0,0 +1,484 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-.
from sqlalchemy import create_engine #, text
from sqlalchemy.engine import URL
from shapely.geometry import Point
from shapely.ops import nearest_points
import pandas as pd
import geopandas as gpd
from pycen import con_sicen as con
numerisateur = ['GEIER Colas']
lib_etude = 'Echange de données (partenaires)'
# précision possible :
# GPS / 0 à 10m / 10 à 100m / 100 à 500m / commune / lieu-dit
precision_gps = 'GPS'
dict_sp = {
'Dilatata dilatata (Gould, 1841)':'Menetus dilatatus (A.A. Gould, 1841)',
'Gomphocerippus biguttulus biguttulus (Linnaeus, 1758)':'Chorthippus biguttulus biguttulus (Linnaeus, 1758)',
'Gomphocerippus mollis mollis (Charpentier, 1825)':'Chorthippus mollis mollis (Charpentier, 1825)',
}
dict_columns = {
'TAXO_LATIN_C':'nom_latin',
'RELV_NOM':'lot_donnees',
'OBSE_DATE_J':'date_obs',
'OBSE_LON':'longitude',
'OBSE_LAT':'latitude',
'SITE_NOM':'localisation',
'OBSE_PLACE':'rmq_loc',
'OBSV_LIBEL':'observateur',
'DETM_LIBEL':'validateur',
'OBSE_COMMENT':'remarque_obs',
'OBSE_PCOLE_CHOI':'protocole',
'OBSE_PRECIS_CHOI':'effectif_textuel',
'OBSE_NOMBRE':'effectif',
'OBSE_TIME':'heure_obs',
'OBSE_COMP_CHOI':'reprostatut',
'OBSE_CARACT_CHOI':'reprocaract',
'OBSE_SIG_OBJ_ID':'id_origine',
'OBSE_NEBUL_CHOI':'nebulosite',
'OBSE_TPTAIR':'temperature',
'OBSE_VENT_CHOI':'vent'
}
# dict_columns = {
# 'Nom scientifique':'nom_latin',
# 'CD_nom':'cd_nom',
# 'Observateur':'observateur',
# 'Date_Relevé':'date_obs',
# 'Code_Relevé':'id_waypoint',
# 'X lambert 93':'longitude',
# 'Y Lambert 93':'latitude',
# 'Altinf_Relevé':'elevation',
# 'Nom_LieuDit':'localisation',
# 'Description_Relevé':'remarque_obs',
# 'Statuts':'remarque_obsStatut',
# 'Lot_données':'lot_donnees',
# 'Localisation_Relevé':'rmq_loc'
# }
# sql = 'SELECT * FROM %s.%s LIMIT 10;' % ('saisie','saisie_observation')
# saisie = gpd.read_postgis(
# sql = sql,
# con = con,
# geom_col = 'geometrie'
# )
def near(point,df2,geom_union,src_column):
# find the nearest point and return the corresponding Place value
geom2_col = df2.geometry.name
nearest = df2[geom2_col] == nearest_points(point, geom_union)[1]
# print(nearest)
# print(df2[nearest][src_column])
return df2[nearest][src_column].values[0]
def get_refGeomTable(table, schema,geom_col='geometrie',colnames=None,mask=None,buffer=None):
'''
Récupération de données géométriques dasn une table de référence
'''
pkey = con.dialect.get_pk_constraint(con,table,schema)['constrained_columns'][0]
if colnames:
colnames = [pkey,*colnames]
else:
colnames = [pkey]
cols = ','.join(colnames)
sql = 'SELECT %s,%s FROM %s.%s' % (cols,geom_col,schema,table)
if mask:
if buffer :
mask = mask.buffer(buffer)
sql = sql + """ WHERE ST_Intersects({geom_col}, 'SRID=2154;{mask}')""".format(
geom_col=geom_col,
mask=mask)
gdf = gpd.read_postgis(
sql,con,
geom_col=geom_col)
return gdf
def get_refTable(table, schema, colnames=None):
pkey = con.dialect.get_pk_constraint(con,table,schema)['constrained_columns'][0]
if colnames:
colnames = [pkey,*colnames]
else:
colnames = [pkey]
sql = 'SELECT %s' % (','.join(colnames))
sql = sql + ' FROM %s.%s' % (schema,table)
df = pd.read_sql(sql,con)
return df
def create_geometry(x,y):
geom = [Point(xy) for xy in zip(x, y)]
return geom
def normalize_obs(obs,name='first',sep=';'):
'''
:obs: pd.Series. Colonne des observateurs.
:name: str. 'first' si le nom est cité en premier.
'first',None. Défault 'first'.
:sep: str. séparateur de personne dans le cas de
multi-observateur. Défault ';'.
'''
obs = obs.replace(sep,',',regex=True)
obs = obs.replace(' ,',',',regex=True)
obs = obs.replace(', ',',',regex=True)
obs = obs.str.split(',',expand=True).stack()
obs = obs.str.split(' ',expand=True)
if name == 'first':
obs[0] = obs[0].str.upper()
obs[1] = obs[1].str[0].str.upper() + obs[1].str[1:]
obs['pers'] = obs[0] + ' ' + obs[1]
else:
obs[1] = obs[1].str.upper()
obs[0] = obs[0].str[0].str.upper() + obs[0].str[1:]
obs['pers'] = obs[1] + ' ' + obs[0]
obs = obs['pers'].unstack()
obs['pers'] = obs[0]
if 1 in obs.columns:
obs.loc[~obs[1].isna(),'pers'] = obs.loc[~obs[1].isna(),'pers'] + '&' +\
obs.loc[~obs[1].isna(),1]
return obs['pers']
def get_pers(obs):
'''
:obs: pd.Series. Colonne des observateurs.
'''
obs = pd.Series(obs.unique())
obs = obs.str.split(' ',expand=True)
obs_name = "','".join(obs[0].str.upper().tolist())
obs_surn = "','".join(obs[1].tolist())
sql = '''
SELECT
concat(nom,' ',prenom) pers,
id_personne,
id_structure
FROM md.personne
WHERE nom in ('%s')
AND prenom in ('%s')
AND id_structure != 49
;''' % (obs_name,obs_surn)
obs = pd.read_sql(sql,con)
return obs
# import TAXREF
taxref = pd.read_sql_table(
table_name='taxref',
con=con,
schema='inpn'
)
taxref = taxref[[
'regne','phylum','classe','ordre',
'famille','nom_valide','nom_vern',
'nom_complet','cd_nom']]
taxref['nom_latin'] = taxref.nom_complet.str.replace(' \(\#ref\)','',regex=True).copy()
# taxref['nom_latin2'] = taxref.nom_latin.str.split(r' \(|, [0-9]',expand=True)[0]
tmp = taxref[
(taxref.nom_latin=='Euconulus Reinhardt, 1883')&
(taxref.cd_nom!='192372')].index
taxref.drop(tmp,inplace=True)
# import DATA
path = '/home/colas/Documents/9_PROJETS/4_SICEN/IMPORT/'
file = 'Données Le Cheylas GD_2020.xlsx'
# file = 'DonneesMeF.ods'
df = pd.read_excel(path+file)
# df = pd.read_csv(path+file)
# mise en forme DATA
df.index.name = 'id'
df.rename(columns=dict_columns, inplace=True)
keep_col = df.columns[df.columns.isin(dict_columns.values())]
df = df[keep_col]
df.nom_latin.replace(dict_sp,inplace=True)
df['statut_validation'] = 'validée'
# df['lieu_dit'] = df['localisation'].copy()
if 'EFFC_OIS_Nb_Adult_Mâl' in df.columns:
df.loc[~df.EFFC_OIS_Nb_Adult_Mâl.isna(),'type_effectif'] = 'Adulte'
del df['EFFC_OIS_Nb_Adult_Mâl']
if 'id_origine' in df.columns:
df.loc[~df.id_origine.isna(),'id_origine'] = df.loc[~df.id_origine.isna(),'id_origine'].astype(int).astype(str)
# Concaténation des remarques de localisation
if 'rmq_loc' in df.columns:
df.loc[~df.rmq_loc.isna(),'localisation'] = df.loc[~df.rmq_loc.isna(),'localisation'] + \
', ' + df.loc[~df.rmq_loc.isna(),'rmq_loc']
del df['rmq_loc']
# Concaténation des remarques observateurs
if 'remarque_obs' in df.columns:
df.loc[~df.remarque_obs.isna(),'remarque_obs'] = 'Commentaires : ' + \
df.loc[~df.remarque_obs.isna(),'remarque_obs']
if 'remarque_obsStatut' in df.columns:
df.loc[~df.remarque_obsStatut.isna(),'remarque_obs'] = df.loc[~df.remarque_obsStatut.isna(),'remarque_obs'] + \
'; rmq_statut : ' + df.loc[~df.remarque_obsStatut.isna(),'remarque_obsStatut']
del df['remarque_obsStatut']
if 'nebulosite' in df.columns:
df.loc[~df.nebulosite.isna(),'remarque_obs'] = df.loc[~df.nebulosite.isna(),'remarque_obs'] + \
'; nebul : ' + df.loc[~df.nebulosite.isna(),'nebulosite']
del df['nebulosite']
if 'temperature' in df.columns:
df.loc[~df.temperature.isna(),'remarque_obs'] = df.loc[~df.temperature.isna(),'remarque_obs'] + \
'; temp : ' + df.loc[~df.temperature.isna(),'temperature'].astype(str)
del df['temperature']
if 'vent' in df.columns:
df.loc[~df.vent.isna(),'remarque_obs'] = df.loc[~df.vent.isna(),'remarque_obs'] + \
'; ' + df.loc[~df.vent.isna(),'vent']
del df['vent']
if 'id_waypoint' in df.columns:
df.loc[~df.id_waypoint.isna(),'id_waypoint'] = df.loc[~df.id_waypoint.isna(),'id_waypoint']\
.astype(int).astype(str)
if 'precision' not in df.columns:
df['precision'] = 'GPS'
# format date / time
# df['date_obs'] = df['date_obs'].astype(str).astype(int)
# df['date_obs'] = pd.to_datetime(df['date_obs'])
# df['date_obs'].astype('datetime64')
# pd.to_datetime(df['heure_obs'],format='%H:%M:%S')
# df['heure_obs']
# create geometrie
df['geometrie'] = create_geometry(df.longitude, df.latitude)
if all(df.longitude < 10):
epsg = 4326
elif all(df.longitude > 900000):
epsg = 2154
df = df.set_geometry('geometrie', crs=epsg)
if epsg == 4326:
df.to_crs(2154, inplace=True)
elif epsg == 2154:
df['longitude'] = df.to_crs(4326).geometrie.x
df['latitude'] = df.to_crs(4326).geometrie.y
df_union = df.unary_union
# join code_INSEE & Lieu-dit
# commune
com = get_refGeomTable(
'commune', 'ign_bd_topo',
geom_col='geometrie',mask=df_union)
com_union = com.unary_union
# lieu_dit
ldt = get_refGeomTable(
'lieu_dit', 'ign_bd_topo', colnames=['nom'],
geom_col='geometrie',mask=com_union, buffer=1000)
ldt.rename(columns={'id': 'id_lieu_dit'}, inplace=True)
ldt_union = ldt.unary_union
# jointure lieu_dit / communes
if 'lieu_dit' in df.columns:
df['lieu_dit'] = df.lieu_dit.str.lower()
df = pd.merge(df,ldt[['id_lieu_dit','nom']],left_on='lieu_dit',right_on='nom')
del df['nom']
del df['lieu_dit']
else:
df['id_lieu_dit'] = [
near(geom,ldt,ldt_union,'id_lieu_dit')
for geom in df.geometrie]
df = gpd.sjoin(df,com)
del df['index_right']
# get observateur/validateur
df['observateur'] = normalize_obs(df['observateur'], sep='-')
obs = get_pers(df.observateur)
if not obs.pers.is_unique:
raise Exception('duplicate observateur')
ind = obs[(obs.pers == 'GRANGE Benjamin')&(obs.id_structure == 1)].index
obs.drop(ind, inplace=True)
obs['id_personne'] = obs['id_personne'].astype(str)
obs['id_structure'] = obs['id_structure'].astype(str)
df['observateur'] = df['observateur'].replace(
to_replace=[*obs.pers],
value=[*obs.id_personne],
regex=True)
if 'validateur' in df.columns:
val = get_pers(df.validateur)
val['id_personne'] = val['id_personne'].astype(str)
df['validateur'].replace(*obs.pers,*obs.id_personne, inplace=True)
else:
df['validateur'] = df['observateur'].str.split('&',expand=True)[0]
for p in obs.iterrows():
o = p[1]['id_personne']
s = p[1]['id_structure']
df.loc[df.observateur.str.contains(o),'structure'] = s
# get numerisateur
num = get_pers(pd.Series(numerisateur))
df['numerisateur'] = num.id_personne.astype(str)[0]
# get id_lot / id_etude
lot = get_refTable('lot_donnee','md',colnames=['libelle'])
etude = get_refTable('etude','md',colnames=['nom_etude'])
df = pd.merge(df,lot,left_on='lot_donnees',right_on='libelle')
del df['libelle']
del df['lot_donnees']
df['id_etude'] = etude.loc[etude.nom_etude == lib_etude,'id_etude'].values[0]
# get protocole
if 'protocole' in df.columns:
proto = get_refTable('protocole','md',colnames=['libelle'])
df.protocole = df.protocole \
.replace(
to_replace=[
'Capture à vue (à la main, au filet à papillon...)',
"Recherche ciblée (bouses, cadavres d'animaux...)"],
value=['Capture à vue',"Recherche d'indices de présence"])
df = pd.merge(df,proto, left_on='protocole',right_on='libelle')
del df['libelle']
del df['protocole']
# join taxref
if 'cd_nom' in df.columns:
if df['cd_nom'].dtype == int: df['cd_nom'] = df['cd_nom'].astype(str)
if 'nom_latin' in taxref.columns: del taxref['nom_latin']
if 'nom_latin' in df.columns: del df['nom_latin']
df = df.merge(taxref,on='cd_nom')
else:
df.nom_latin = df.nom_latin.str.strip()
tt = taxref[taxref.nom_latin.isin(df.nom_latin.unique())]
len_taxo = df.nom_latin.unique().shape[0]
len_taxref = tt.shape[0]
if len_taxo == len_taxref:
df = df.merge(tt,on='nom_latin')
del df['nom_latin']
else:
contain = '|'.join(df.nom_latin.unique())
tt = taxref[taxref.nom_latin.str.contains(contain)]
len_taxref = tt.shape[0]
if len_taxo == len_taxref:
df = df.merge(tt,on='nom_latin')
del df['nom_latin']
else:
raise Exception('error corespondance cd_nom')
contain = '|'.join(df.nom_latin.str.split(r' \(|, [0-9]',expand=True)[0].unique())
tt = taxref[taxref.nom_latin2.str.contains(contain)]
# get statuts de reproduction
if 'reprostatut' in df.columns:
if 'reprocaract' in df.columns:
df.loc[(~df.reprostatut.isna())&(~df.reprocaract.isna()),'remarque_obs'] = df['remarque_obs'] + \
'; rmq_repro : ' + df.loc[(~df.reprostatut.isna())&(~df.reprocaract.isna()),'reprocaract']
df.loc[df.reprostatut.isna(),'reprostatut'] = df.loc[df.reprostatut.isna(),'reprocaract']
del df['reprocaract']
sql = 'SELECT unnest(enum_range(null::{0}.{1})) {1}'.format('saisie','enum_reprostatut')
reprostat = pd.read_sql(sql,con)
reprostat = pd.concat(
[reprostat,reprostat.enum_reprostatut.str.split('_',expand=True)],
axis=1 )
lst_phyl = df.phylum.unique()
lst_clas = set([*df.classe.unique()])
lst_ordr = set([*df.ordre.unique()])
AMP = set(['Amphibia'])
CHIR = set([])
INV = set(['Bivalvia','Gastropoda'])
MAM = set([])
ODO = set(['Hexapoda'])
ordr_ODO = set(['Odonata'])
ordr_INV = set(['Orthoptera','Lepidoptera'])
OIS = set(['Aves'])
REP = set([])
if any(lst_clas.intersection(AMP)):
# AMP = list(AMP)
repro = reprostat[reprostat[0]=='AMP']
df.loc[df.classe.isin(AMP),'reprostatut'] = df[df.classe.isin(AMP)].reprostatut \
.replace(
to_replace='Reproduction confirmée',
value='Accouplement') \
.replace(to_replace=[*repro[1]],value=[*repro.enum_reprostatut])
# df.loc[df.classe.isin(AMP),'reprostatut'] = df[df.classe.isin(AMP)].reprostatut \
# .replace(to_replace=[*repro[1]],value=[*repro.enum_reprostatut])
if any(lst_clas.intersection(CHIR)):
# CHIR = list(CHIR)
repro = reprostat[reprostat[0]=='CHIR']
if any(lst_clas.intersection(MAM)):
# MAM = list(MAM)
repro = reprostat[reprostat[0]=='MAM']
if any(lst_clas.intersection(INV)):
# INV = list(INV)
repro = reprostat[reprostat[0]=='INV']
df.loc[df.classe.isin(INV),'reprostatut'] = df[df.classe.isin(INV)].reprostatut \
.replace(
to_replace='Reproduction confirmée',
value='Accouplement') \
.replace(to_replace=[*repro[1]],value=[*repro.enum_reprostatut])
# df.loc[df.classe.isin(INV),'reprostatut'] = df[df.classe.isin(INV)].reprostatut \
# .replace(to_replace=[*repro[1]],value=[*repro.enum_reprostatut])
if any(lst_clas.intersection(ODO)) and any(lst_ordr.intersection(ordr_INV)):
# ODO = list(ODO)
# ordr_INV = list(ordr_INV)
repro = reprostat[reprostat[0]=='INV']
df.loc[(df.classe.isin(ODO))&(df.ordre.isin(ordr_INV)),'reprostatut'] = \
df[(df.classe.isin(ODO))&(df.ordre.isin(ordr_INV))].reprostatut \
.replace(
to_replace='Reproduction confirmée',
value='Accouplement') \
.replace(to_replace=[*repro[1]],value=[*repro.enum_reprostatut])
# df.loc[(df.classe.isin(ODO))&(df.ordre.isin(ordr_INV)),'reprostatut'] = \
# df[(df.classe.isin(ODO))&(df.ordre.isin(ordr_INV))].reprostatut \
# .replace(to_replace=[*repro[1]],value=[*repro.enum_reprostatut])
# ODO = set(ODO)
if any(lst_clas.intersection(ODO)) and any(lst_ordr.intersection(ordr_ODO)):
# ODO = list(ODO)
# ordr_ODO = list(ordr_ODO)
repro = reprostat[reprostat[0]=='ODO']
df.loc[(df.classe.isin(ODO))&(df.ordre.isin(ordr_ODO)),'reprostatut'] = \
df[(df.classe.isin(ODO))&(df.ordre.isin(ordr_ODO))].reprostatut \
.replace(
to_replace=['Accouplement','Reproduction confirmée','Reproduction possible','Reproduction probable'],
value=['Tandem','Exuvie/émergence','Mâles+Femelles','Immature']) \
.replace(to_replace=[*repro[1]],value=[*repro.enum_reprostatut])
# df.loc[(df.classe.isin(ODO))&(df.ordre.isin(ordr_ODO)),'reprostatut'] = \
# df[(df.classe.isin(ODO))&(df.ordre.isin(ordr_ODO))].reprostatut \
# .replace(to_replace=[*repro[1]],value=[*repro.enum_reprostatut])
if any(lst_clas.intersection(OIS)):
# OIS = list(OIS)
repro = reprostat[reprostat[0]=='OIS']
df.loc[df.classe.isin(OIS),'reprostatut'] = df[df.classe.isin(OIS)].reprostatut \
.replace(
to_replace='Déplacement (marche, vol, nage)',
value='Fuite - envol') \
.replace(to_replace=[*repro[1]],value=[*repro.enum_reprostatut])
# df.loc[df.classe.isin(OIS),'reprostatut'] = df[df.classe.isin(OIS)].reprostatut \
# .replace(to_replace=[*repro[1]],value=[*repro.enum_reprostatut])
if any(lst_clas.intersection(REP)):
# REP = list(REP)
repro = reprostat[reprostat[0]=='REP']
df.to_postgis(
name='saisie_observation',
con=con,
schema='saisie',
if_exists='append',
index=False,
geom_col='geometrie',
)
1_SICEN/refresh_vm.py
+12
View File
@@ -0,0 +1,12 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from pycen import con_sicen
def refresh_vm():
sql = "REFRESH MATERIALIZED VIEW saisie.vm_synthese_observations WITH DATA;"
with con_sicen.begin() as cnx:
cnx.execute(sql)
if __name__ == "__main__":
refresh_vm()
1_SICEN/sicen2_clean.py
+410
View File
@@ -0,0 +1,410 @@
from os import replace
import pandas as pd
import numpy as np
from geoalchemy2 import Geometry
# import numpy as np
from sqlalchemy import create_engine, text
from sqlalchemy.engine import URL
import geopandas as gpd
from pycen import con_sicen as bckup_con
# con = create_engine('postgresql+psycopg2://{1}:{0}@{2}:{3}/{4}'.format(pwd, user, adr, port, base), echo=False)
# bckup_con = create_engine('postgresql+psycopg2://{1}:{0}@{2}:{3}/{4}'.format(bckup_pwd, bckup_user, bckup_adr, bckup_port, bckup_base), echo=False)
def update_data(df, con, sch, tab, epsg=None):
columns = df.columns.to_list()
frame = df.copy()
frame.replace("'","''", regex=True, inplace=True)
pkey = con.dialect.get_pk_constraint(con, table_name=tab, schema=sch)['constrained_columns']
for col in frame.select_dtypes(include=['object']).columns:
test = frame[frame[col].str.contains("'", na=False)]
if not test.empty:
frame[col].replace("'", "''", inplace=True, regex=True)
# print(frame[col])
if 'geom' in columns or 'geometry' in columns:
if epsg or df.crs:
if not epsg:
epsg = df.crs.to_epsg()
name_geom = df.geometry.name
frame[name_geom] = 'SRID={epsg};'.format(epsg=epsg) + df[name_geom].map(str)
# else: return 'No crs define in update_data or in gdf'
for c, col in enumerate(columns):
if c == 0:
frame['insert'] = "('" + frame[col].map(str)
# break
else:
frame['insert'] = frame['insert'] + "','" + frame[col].map(str)
if c == len(columns)-1:
frame['insert'] = frame['insert'] + "')"
# if c == 0:
# frame['insert'] = '("' + frame[col].map(str)
# # break
# else:
# frame['insert'] = frame['insert'] + '","' + frame[col].map(str)
# if c == len(columns)-1:
# frame['insert'] = frame['insert'] + '")'
lst_cols = ', '.join(columns)
lst_vals = ','.join(frame['insert'])
lst_vals.replace("\'None\'", 'NULL', inplace=True)
lst_dupKey = ', '.join([col + '=EXCLUDED.' + col for col in columns])
lst_pkey = ','.join(pkey)
sql = '''INSERT INTO {sch}.{tab} ({lst_cols}) VALUES {lst_vals} ON CONFLICT ({lst_pkey}) DO UPDATE SET {lst_dupKey} ;'''.format(
sch=sch, tab=tab, lst_cols=lst_cols, lst_vals=lst_vals, lst_dupKey=lst_dupKey, lst_pkey=lst_pkey)
# sql = '''INSERT INTO {sch}.{tab} ({lst_cols})
# VALUES {lst_vals}
# ON CONFLICT DO NOTHING;
# '''.format(sch=sch, tab=tab, lst_cols=lst_cols, lst_vals=lst_vals)
try:
con.execute(sql)
# con.execute(text(sql))
print('''
Update OK !''')
except Exception as exept:
print(exept)
df = pd.read_sql_table(
table_name = 'personne',
con = con,
schema = 'md',
index_col='id_personne')
tmp = pd.DataFrame(df.nom.str.split(' ',1).tolist(), columns = ['nom','prenom'], index=df.index)
tmp = tmp[~tmp.prenom.isnull()]
df.loc[tmp.index,'prenom'] = tmp['prenom']
df.loc[tmp.index,'nom'] = tmp['nom']
df['nom'] = [nom.replace('CEN38_','') for nom in df['nom']]
df['nom'] = [nom.replace('GENTIANA_','') for nom in df['nom']]
df['nom'] = [nom.replace('LPO38_','') for nom in df['nom']]
df['nom'] = [nom.replace('LPO_','') for nom in df['nom']]
df['nom'] = [nom.replace('CENRA_','') for nom in df['nom']]
df['nom'] = [nom.replace('GRPLS_','') for nom in df['nom']]
rm_row = ['ISÈRE', 'Ecrin', 'FUR', 'BRIQUIR', 'BORGNE', '(FMBDS)', 'Isère', 'Rhône', '(Presonnel)', 'Monsieur', 'Batrachologique de France', '(Ecosphère)',
'PIC VERT Association', 'BAGOUSSE', '(BIOTOPE)', '(Dauphinelle)', 'Cras', '(GRPLS)', 'et Vie Sociale', '(ONCFS)', 'campagne sauvetage amphibiens']
df = df[~df.prenom.isin(rm_row)]
tmp = pd.DataFrame(df.prenom.str.split(' ',1).tolist(), columns = ['nom','prenom'], index=df.index)
tmp = tmp[~tmp.prenom.isnull()]
tmp.drop(index=[100032,8628,8645,4238,8058,8070,8353,1099,1081,1323,1324], inplace=True)
df.loc[tmp.index,'nom'] = df.loc[tmp.index,'nom'] + ' ' + tmp['nom']
df.loc[tmp.index,'prenom'] = tmp['prenom']
tmp = pd.DataFrame(
df.loc[df.nom=='Abbé',['nom','prenom']].prenom.str.split(' ',1).tolist(),
columns = ['prenom','nom'],
index=df.loc[df.nom=='Abbé',['nom','prenom']].index)
tmp2 = df.loc[df.nom=='Abbé',['nom','prenom']]
df.loc[tmp.index,'nom'] = tmp['nom']
df.loc[tmp.index,'prenom'] = tmp2['nom'] + ' ' + tmp['prenom']
df.reset_index(inplace=True)
tmp = df.copy()
tmp = tmp[['id_personne','nom','prenom', 'id_structure']]
# update_data(tmp,con,sch='md',tab='personne')
id_old = '8044'
id_new = '1000014'
sql='''SELECT * FROM saisie.saisie_observation WHERE observateur = '{0}' OR validateur = {0};'''.format(id_old)
# sql = 'SELECT nom, prenom FROM md.personne WHERE id_personne= 6191;'
pd.read_sql(
sql=sql,
con=con
)[['observateur','validateur']]
# df[['prenom', 'nom']].values.tolist()
# sql = '''UPDATE saisie.saisie_observation SET validateur = 100077
# WHERE validateur = 1110;'''
try:
con.execute(sql)
# con.execute(text(sql))
print('''
Update OK !''')
except Exception as exept:
print(exept)
tmpx = df.copy()
tmpx = tmpx[['id_personne','nom','prenom','id_structure']]
tutux = tmpx.drop_duplicates(['nom','prenom','id_structure'], keep=False)
tomx = tmpx[~tmpx.id_personne.isin(tutux.id_personne)]
tomx = tomx.sort_values('nom')
for nom in tomx.nom.unique().tolist():
tmp1 = tomx[tomx.nom==nom]
keep = max(tmp1.id_personne)
rep = min(tmp1.id_personne)
sql = '''UPDATE saisie.saisie_observation SET observateur = '{keep}' WHERE observateur = '{replace}';'''.format(
keep=keep, replace=rep
)
# sql = '''UPDATE saisie.saisie_observation SET validateur = 100077
# WHERE validateur = 1110;'''
try:
con.execute(sql)
# con.execute(text(sql))
print('''
Update OK !''')
except Exception as exept:
print(exept)
sql='''SELECT * FROM saisie.saisie_observation WHERE observateur = '{replace}' OR validateur = {replace};'''.format(
replace=rep
)
# sql = 'SELECT nom, prenom FROM md.personne WHERE id_personne= 6191;'
test = pd.read_sql(
sql=sql,
con=con
)[['observateur','validateur']]
if test.empty:
print(''' OK for %s'''%nom)
else:
print('ERROR !!!!!!!!!')
break
print(nom)
sql='''SELECT * FROM md.personne WHERE nom like '{nom}%';'''.format(nom=nom)
tutu = pd.read_sql(
sql=sql,
con=con
)
if tutu.shape[0] > 1:
df
print('''
{id_personne},{nom},{prenom},{id_structure}'''.format(
id_personne=id_personne,nom=nom,prenom=prenom,id_structure=id_structure))
from sqlalchemy import create_engine, text
from sqlalchemy.engine import URL
from shapely.geometry import Point
import matplotlib.pyplot as plt
import contextily as cx
import geopandas as gpd
import pandas as pd
from pycen import con_sicen as con
df1 = pd.read_sql_table(
table_name = 'saisie_observation',
con = con,
schema = 'saisie',
index_col='id_obs')
geometry = [Point(xy) for xy in zip(df1['longitude'], df['latitude'])]
crs = 'epsg:4326'
df = gpd.GeoDataFrame(df1, crs=crs, geometry=geometry)
df = df.to_crs(epsg=3857)
df_unkeep = df.drop_duplicates(keep=False)
df_clean = df.drop_duplicates(keep='first')
df[~df.index.isin(df_clean.index)].sort_values('date_obs').index.values
df[~df.index.isin(df_unkeep.index)].sort_values('date_obs').index.values
ax = df[~df.index.isin(df_unkeep.index)][['geometry']].plot()
cx.add_basemap(ax)
plt.show()
df.sort_index(inplace=True)
data_drop = df[~df.index.isin(df.drop_duplicates(keep=False).index)]
tmp = [i.split('&') for i in df['observateur'].unique() ]
lst_pers = [item for sublist in tmp for item in sublist]
df_pers = pd.DataFrame(lst_pers, columns=['id_pers']).drop_duplicates()
sql='''SELECT * FROM md.personne WHERE id_personne IN ({ids});'''.format(ids=','.join(df_pers.id_pers))
tutu = pd.read_sql(
sql=sql,
con=con
)
df_pers['id_pers'] = df_pers['id_pers'].astype(int)
df_pers[~df_pers.id_pers.isin(tutu.id_personne)]
tom = toto.copy()
tom = tom[tom.observateur.str.contains('&')]
tom['observateur'] = [idx.replace('1043','100086') for idx in tom['observateur']]
tom['observateur'] = [idx.replace('6191','1000002') for idx in tom['observateur']]
tom['observateur'] = [idx.replace('6051','1000013') for idx in tom['observateur']]
tom['observateur'] = [idx.replace('1070','1000011') for idx in tom['observateur']]
tom['observateur'] = [idx.replace('1088','100043') for idx in tom['observateur']]
tom['observateur'] = [idx.replace('6045','1000004') for idx in tom['observateur']]
tom['observateur'] = [idx.replace('6121','1000009') for idx in tom['observateur']]
tom['observateur'] = [idx.replace('6355','1000003') for idx in tom['observateur']]
tom['observateur'] = [idx.replace('8632','1000005') for idx in tom['observateur']]
tom['observateur'] = [idx.replace('1034','100007') for idx in tom['observateur']]
tom['observateur'] = [idx.replace('1110','100077') for idx in tom['observateur']]
tom['observateur'] = [idx.replace('1075','1000022') for idx in tom['observateur']]
tom['observateur'] = [idx.replace('1188','100038') for idx in tom['observateur']]
tom['observateur'] = [idx.replace('8271','100017') for idx in tom['observateur']]
tom['observateur'] = [idx.replace('1049','100053') for idx in tom['observateur']]
tom['observateur'] = [idx.replace('8328','100034') for idx in tom['observateur']]
tom['observateur'] = [idx.replace('1102','100062') for idx in tom['observateur']]
tom['observateur'] = [idx.replace('1016','1000001') for idx in tom['observateur']]
tom['observateur'] = [idx.replace('1187','100037') for idx in tom['observateur']]
tom['observateur'] = [idx.replace('8018','1000008') for idx in tom['observateur']]
tom['observateur'] = [idx.replace('4233','100067') for idx in tom['observateur']]
tom['observateur'] = [idx.replace('8254','1000012') for idx in tom['observateur']]
tom['observateur'] = [idx.replace('8015','100016') for idx in tom['observateur']]
df.sort_index(inplace=True)
tom.sort_index(inplace=True)
df.loc[df.index.isin(tom.index), 'observateur'] = tom['observateur']
df.reset_index(inplace=True)
tmp = df.copy()
tmp = tmp[['id_obs','observateur']]
update_data(tmp,con,sch='saisie',tab='saisie_observation')
# Check user in suivi_saisie_observation existent tous!
from datetime import datetime as dt
df = pd.read_sql_table(
table_name = 'suivi_saisie_observation',
con = con,
schema = 'saisie',
)
df.sort_values('date_operation', ascending=False, inplace=True)
x = '2021-05-25 22:00:00'
y = '2021-05-21 12:30:00'
x = dt.strptime(x, '%Y-%m-%d %H:%M:%S')
y = dt.strptime(y, '%Y-%m-%d %H:%M:%S')
ddf = df[(df.date_operation < x) & (df.date_operation > y)]
ddf = ddf[['operation','utilisateur', 'id_obs', 'observateur', 'validateur', 'numerisateur']]
# Get all "id" observateur
tmp_obs = [i.split('&') for i in ddf['observateur'].dropna().unique() ]
lst_pers_obs = [item for sublist in tmp_obs for item in sublist]
tmp_val = [i.split('&') for i in ddf['validateur'].dropna().unique().astype(int).astype(str) ]
lst_pers_val = [item for sublist in tmp_val for item in sublist]
tmp_num = [i.split('&') for i in ddf['numerisateur'].dropna().unique().astype(str) ]
lst_pers_num = [item for sublist in tmp_num for item in sublist]
lst_pers = lst_pers_obs + lst_pers_val + lst_pers_num
df_pers = pd.DataFrame(lst_pers, columns=['id_pers']).drop_duplicates()
# Check if personnes exist
pers_not_exist = []
for i in df_pers.id_pers:
sql = 'SELECT id_personne,prenom, nom, id_structure FROM md.personne WHERE id_personne = %s;'%i
res = pd.read_sql(
sql = sql,
con = con
)
if res.empty:
print(i)
pers_not_exist += [i]
sql = '''SELECT * FROM saisie.suivi_saisie_observation
WHERE utilisateur = 'inconnu'
AND date_operation < '2021-05-25 22:00:00'
AND date_operation > '2021-05-21 12:30:00';'''
pd.read_sql(
sql=sql,
con=con
)
sql = '''UPDATE saisie.suivi_saisie_observation SET utilisateur = 'colas.geier@cen-isere.org'
WHERE utilisateur = 'inconnu'
AND date_operation < '2021-05-25 22:00:00'
AND date_operation > '2021-05-21 12:30:00';'''
try:
con.execute(sql)
# con.execute(text(sql))
print('''
Update OK !''')
except Exception as exept:
print(exept)
rep = '8044'
keep = '1000014'
sql_check ='''SELECT * FROM saisie.saisie_observation WHERE observateur like '%{replace}%' OR validateur = {replace};'''.format(
replace=rep
)
sql_check ='''SELECT * FROM saisie.saisie_observation WHERE observateur = '{replace}' OR validateur = {replace};'''.format(
replace=rep
)
sql_updt = '''
UPDATE saisie.saisie_observation SET observateur = '{keep}' WHERE observateur = '{replace}';
UPDATE saisie.saisie_observation SET numerisateur = {keep} WHERE numerisateur = {replace};
'''.format(
keep=keep, replace=rep
)
# sql = '''UPDATE saisie.saisie_observation SET validateur = 100077
# WHERE validateur = 1110;'''
test = pd.read_sql(
sql=sql_check,
con=con
)[['observateur','validateur']]
if test.empty:
print(''' OK for %s in table saisie'''%nom)
else:
try:
con.execute(sql_updt)
# con.execute(text(sql))
print('''
Update OK !''')
except Exception as exept:
print(exept)
sql_check2 ='''SELECT * FROM saisie.suivi_saisie_observation WHERE observateur = '{replace}' OR validateur = '{replace}';'''.format(
replace=rep
)
sql_updt2 = '''
UPDATE saisie.suivi_saisie_observation SET observateur = '{keep}' WHERE observateur = '{replace}';
UPDATE saisie.suivi_saisie_observation SET numerisateur = {keep} WHERE numerisateur = {replace};
'''.format(
keep=keep, replace=rep
)
test2 = pd.read_sql(
sql=sql_check2,
con=con
)[['observateur','validateur']]
if test2.empty:
print(''' OK for %s in table saisie'''%nom)
else:
try:
con.execute(sql_updt2)
# con.execute(text(sql))
print('''
Update OK !''')
except Exception as exept:
print(exept)
sql='''SELECT * FROM saisie.saisie_observation WHERE observateur = '{replace}' OR validateur = {replace};'''.format(
replace=rep
)
# sql = 'SELECT nom, prenom FROM md.personne WHERE id_personne= 6191;'
test = pd.read_sql(
sql=sql,
con=con
)[['observateur','validateur']]
if test.empty:
print(''' OK for %s'''%nom)
else:
print('ERROR !!!!!!!!!')
# break
1_SICEN/sicen2_extract.py
+81
View File
@@ -0,0 +1,81 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-.
import geopandas as gpd
import pandas as pd
from geoalchemy2 import Geometry
from sqlalchemy.engine import URL
from sqlalchemy import create_engine
import sys
# Liste des sites
lst_site = ['GOUT']
# Suppression des champs souhaités
delete = True
lst_cols_del = ['geometrie','date_debut_obs']
# Format d'extraction ['xls', 'xlsx', 'csv']
FORMAT = 'xlsx'
# Localistation du fichier de sortie
OUTPUT = '~/Documents/tmp/Jean_Luc/'
# Nom du fichier de sortie
NAME_OUT = 'observation_site_GOUT'
from pycen import con_sicen as engine_hom
# Parametres bdd HOMER (out)
schema_hom = 'saisie'
table_hom = 'saisie_observation'
# engine_hom = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user_hom,pwd_hom,adr_hom,port_hom,base_hom), echo=False)
# Parametres bdd BART (in)
# BART ETEINT : OBSOLETE
adr_bart = '192.168.0.3'
port_bart = '5432'
base_bart = 'bd_cen38'
schema_bart = 'sites'
table_bart = "c_sites_zonages"
url_bart = URL.create('postgresql+psycopg2',
username=user_bart,
password=pwd_bart,
host=adr_bart,
database=base_bart,
)
con_bart = create_engine(url_bart)
# con_bart = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user_bart,pwd_bart,adr_bart,port_bart,base_bart), echo=False)
epsg = '2154'
crs = 'EPSG:%s'%epsg
sql = "SELECT geom FROM {sch}.{tab} WHERE code_site in ('{lst_site}') AND type_zonage = 'ZO'".format(sch=schema_bart, tab=table_bart, lst_site="','".join(lst_site))
gdf_site = gpd.read_postgis(
sql = sql,
con = con_bart,
)
if gdf_site.shape[0] > 1 :
union = gdf_site.geom.cascaded_union
Sunion = gpd.GeoSeries(union)
gdf_union = gpd.GeoDataFrame(Sunion, geometry='geom', columns=['geom'], crs=crs)
gdf_site = gdf_union
sql = '''SELECT * FROM {sch}.{tab} WHERE ST_Intersects (geometrie, 'SRID={epsg};{poly}')'''.format(sch=schema_hom, tab=table_hom, epsg=epsg, poly=gdf_site.geom[0])
gdf_saisie = gpd.read_postgis(
sql = sql,
con = engine_hom,
geom_col = 'geometrie'
)
df = pd.DataFrame(gdf_saisie)
if delete:
df.drop(columns= lst_cols_del, inplace=True)
if FORMAT in ['xls', 'xlsx']:
df.to_excel(OUTPUT + NAME_OUT + '.' + FORMAT)
if FORMAT == 'csv':
df.to_file(OUTPUT + NAME_OUT + '.' + FORMAT)
sys.exit('END')
1_SICEN/taxonAbsentTaxref_TO_taxref.py
+24
View File
@@ -0,0 +1,24 @@
#!/usr/bin/env python3
# -*- coding:UTF-8 -*-
from sqlalchemy import create_engine #, text
from sqlalchemy.engine import URL
import pandas as pd
from pycen import con_sicen as con
sql = '''
SELECT * FROM inpn.taxons_isere_absents_taxref
WHERE cd_nom::text NOT IN (SELECT cd_nom FROM inpn.taxref)'''
taxabs = pd.read_sql_query(
sql=sql,con=con
)
taxabs.cd_ref = taxabs.cd_ref.astype(int).astype(str)
taxabs.drop(columns=['id_taxon','source'], inplace=True)
taxabs.to_sql(
name='taxref',
con=con,
schema='inpn',
if_exists='append',
index=False
)
1_SICEN/update_codeInsee.py
+79
View File
@@ -0,0 +1,79 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-.
from sqlalchemy import create_engine, text
from sqlalchemy.engine import URL
from datetime import datetime as dt
from pycen import update_to_sql
import pandas as pd
import geopandas as gpd
from sqlalchemy.sql.expression import column
nom_numerateur = 'GEIER'
pnom_numerator = 'Colas'
today = dt.now().date().isoformat()
# Parametres bdd
from pycen import con_sicen as con
# FONCTION
def get_numerator(nom,pnom,id_structure=1):
nom_numerateur = nom.upper()
pnom_numerator = pnom[0].upper() + pnom[1:]
query_get_numerator = """
SELECT * FROM md.personne
WHERE nom = '%s'
AND prenom = '%s'
AND id_structure = %i
""" % (nom_numerateur,pnom_numerator,id_structure)
numerateur = pd.read_sql_query(
sql = query_get_numerator,
con = con
)
return numerateur
# SELECT OBSERVATION
sql = 'SELECT * FROM saisie.saisie_observation'
saisie = gpd.read_postgis(
sql,
con,
'geometrie')
saisie.sort_values('id_obs', inplace=True)
# SELECT COMMUNE
sql = 'SELECT * FROM ign_bd_topo.commune'
com = gpd.read_postgis(
sql,
con,
'geometrie')
# JOINTURE spatiale
df = gpd.sjoin(saisie[['id_obs','code_insee','geometrie']],com[['code_insee','geometrie']],op='within')
df.sort_values('id_obs', inplace=True)
tmp = df[~df.code_insee_left.eq(df.code_insee_right)].copy()
tmp.rename(columns={'code_insee_right':'code_insee'}, inplace=True)
# UPDATE SET new code_insee
update_to_sql(
tmp[['id_obs','code_insee']],
con=con,
table_name='saisie_observation',
schema_name='saisie',
key_name='id_obs',
geom_col='geometrie'
)
# UPDATE utilisateur OF saisie.suivi_saisie_observation
numerateur = get_numerator(nom_numerateur,pnom_numerator)
update_structure_name = """
UPDATE saisie.suivi_saisie_observation a
SET utilisateur = REPLACE(utilisateur, 'inconnu', '{email_numerator}')
WHERE date_operation > '{today}'
;
""".format(
email_numerator=numerateur.email.item(),
today = today
)
with con.begin() as cnx:
cnx.execute(update_structure_name)
1_SICEN/update_dataONsicen.py
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from sqlalchemy import create_engine
from sqlalchemy.engine import URL
from datetime import datetime as dt
import pandas as pd
import pycen
# Parametres bdd
from pycen import con_sicen as con
sch = 'saisie'
tab = 'saisie_observation'
ids = 'id_obs'
col_updt = 'phenologie'
sql = """
SELECT
{ids},
{col}
FROM {sch}.{tab}
WHERE {col} = 'Inderterminé'
""".format(sch=sch,tab=tab,ids=ids,col=col_updt)
df = pd.read_sql_query(sql,con)
df[col_updt] = 'Indéterminé'
pycen.update_to_sql(df,con,tab,sch,ids,geom_col=None)
tab_suivi = 'suivi_saisie_observation'
ids_suivi = ['operation','date_operation','id_obs']
col_updtSuivi = 'utilisateur'
date = dt.today().date().isoformat()
sql = """
SELECT
{ids},
{col}
FROM {sch}.{tab}
WHERE {col} = 'inconnu'
AND date_operation > '{date}'
""".format(sch=sch,tab=tab_suivi,ids=','.join(ids_suivi),col=col_updtSuivi, date=date)
df = pd.read_sql_query(sql,con)
df[col_updtSuivi] = 'colas.geier@cen-isere.org'
pycen.update_to_sql(
df,
con,
table_name=tab_suivi,
schema_name=sch,
key_name = ids_suivi,
geom_col=None)
1_SICEN/update_useroperator_suivisaisie.py
+43
View File
@@ -0,0 +1,43 @@
from pycen import con_sicen as con
import pandas as pd
from datetime import datetime as dt
# FONCTION
def get_numerator(nom,pnom,id_structure=1):
nom_numerateur = nom.upper()
pnom_numerator = pnom[0].upper() + pnom[1:]
query_get_numerator = """
SELECT * FROM md.personne
WHERE nom = '%s'
AND prenom = '%s'
AND id_structure = %i
""" % (nom_numerateur,pnom_numerator,id_structure)
numerateur = pd.read_sql_query(
sql = query_get_numerator,
con = con
)
return numerateur
if __name__ == "__main__":
nom_numerateur = 'GEIER'
pnom_numerator = 'Colas'
today = dt.now().date().isoformat()
# UPDATE utilisateur OF saisie.suivi_saisie_observation
numerateur = get_numerator(nom_numerateur,pnom_numerator)
update_structure_name = """
UPDATE saisie.suivi_saisie_observation a
SET utilisateur = REPLACE(utilisateur, 'inconnu', '{email_numerator}')
WHERE date_operation > '{today}'
;
""".format(
email_numerator=numerateur.email.item(),
today = today
)
with con.begin() as cnx:
cnx.execute(update_structure_name)
print('END UPDATE utilisateur OF saisie.suivi_saisie_observation')
1_SICEN/v_saisie_.py
+273
View File
@@ -0,0 +1,273 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
import geopandas as gpd
from pycen import con_sicen
drop_v_saisie_ = "DROP VIEW IF EXISTS saisie.v_saisie_observation"
v_saisie_observation = """
CREATE OR REPLACE VIEW saisie.v_saisie_observation
AS WITH auteur AS (
SELECT p_1.id_personne,
CASE
WHEN p_1.prenom ~~ '(%%'::text THEN p_1.nom
ELSE COALESCE((p_1.nom || ' '::text) || p_1.prenom)
END AS personne,
s_1.nom_structure AS structure
FROM md.personne p_1
JOIN md.structure s_1 USING (id_structure)
)
SELECT s.id_obs,
s.date_obs,
s.date_debut_obs,
s.date_fin_obs,
s.date_textuelle,
s.regne,
s.nom_vern,
s.nom_complet,
s.cd_nom,
t.cd_ref,
s.effectif_textuel,
s.effectif_min,
s.effectif_max,
CASE
WHEN s.regne = 'Plantae'::text THEN s.type_effectif
ELSE NULL::text
END AS strate_flore,
CASE
WHEN s.regne = 'Plantae'::text THEN s.phenologie
ELSE NULL::text
END AS phenologie_flore,
CASE
WHEN s.regne = 'Animalia'::text THEN s.type_effectif
ELSE NULL::text
END AS age_faune,
CASE
WHEN s.regne = 'Animalia'::text THEN s.phenologie
ELSE NULL::text
END AS sexe_faune,
s.id_waypoint,
s.longitude,
s.latitude,
s.localisation,
md.liste_nom_auteur(s.observateur::character varying) AS observateur,
a1.personne AS numerisateur,
a2.personne AS validateur,
md.liste_nom_structure(s.structure::character varying) AS structure,
s.remarque_obs,
s.code_insee,
s.id_lieu_dit,
s.diffusable,
s."precision",
s.statut_validation,
e.nom_etude AS etude,
p.libelle AS protocole,
s.effectif,
s.url_photo,
s.commentaire_photo,
s.decision_validation,
s.heure_obs,
s.determination,
s.elevation,
s.geometrie,
s.phylum,
s.classe,
s.ordre,
s.famille,
s.nom_valide,
s.qualification,
s.reprostatut AS reprostatut_faune,
s.obs_null,
s.uuid,
ld.libelle AS lot_donnee,
s.id_origine
FROM saisie.saisie_observation s
LEFT JOIN auteur a1 ON a1.id_personne = s.numerisateur
LEFT JOIN auteur a2 ON a2.id_personne = s.validateur
LEFT JOIN md.etude e ON s.id_etude = e.id_etude
LEFT JOIN md.protocole p ON s.id_protocole = p.id_protocole
LEFT JOIN md.lot_donnee ld ON s.id_lot = ld.id_lot
LEFT JOIN inpn.taxref t using (cd_nom)
;
"""
grant_v_saisie = """
ALTER TABLE saisie.v_saisie_observation OWNER TO cgeier;
GRANT ALL ON TABLE saisie.v_saisie_observation TO cgeier;
"""
with con_sicen.begin() as cnx:
cnx.execute(drop_v_saisie_)
cnx.execute(v_saisie_observation)
cnx.execute(grant_v_saisie)
drop_vm_synthese = "DROP MATERIALIZED VIEW IF EXISTS saisie.vm_synthese_observations CASCADE;"
vm_synthese_observations = """
-- saisie.vm_synthese_observations source
CREATE MATERIALIZED VIEW saisie.vm_synthese_observations
TABLESPACE pg_default
AS WITH
observateurs AS (
SELECT personne.id_personne,
(personne.nom || ' '::text) || personne.prenom AS nom_complet,
personne.role,
personne.id_structure
FROM md.personne
),
structures AS (
SELECT structure.id_structure,
structure.nom_structure,
structure.diffusable
FROM md.structure
)
SELECT DISTINCT s.id_obs,
e.nom_etude AS etude,
p.libelle AS protocole,
ld.libelle AS lot_donnee,
s.date_obs,
s.date_debut_obs,
s.date_fin_obs,
s.heure_obs,
CASE
WHEN tx.regne IS NOT NULL THEN tx.regne
ELSE ta.regne
END AS regne,
--tx.phylum,
CASE
WHEN tx.phylum IS NOT NULL THEN tx.phylum
ELSE ta.phylum
END AS phylum,
--tx.classe,
CASE
WHEN tx.classe IS NOT NULL THEN tx.classe
ELSE ta.classe
END AS classe,
--tx.ordre,
CASE
WHEN tx.ordre IS NOT NULL THEN tx.ordre
ELSE ta.ordre
END AS ordre,
tx.famille,
CASE
WHEN tx.group1_inpn IS NOT NULL THEN tx.group1_inpn
ELSE ta.group1_inpn
END AS group1_inpn,
CASE
WHEN tx.group2_inpn IS NOT NULL THEN tx.group2_inpn
ELSE ta.group2_inpn
END AS group2_inpn,
CASE
WHEN tx.lb_nom IS NOT NULL THEN tx.lb_nom
ELSE ta.lb_nom
END AS nom_latin,
CASE
WHEN tx.nom_vern IS NOT NULL THEN tx.nom_vern
ELSE ta.lb_nom
END AS nom_vern,
CASE
WHEN tx.nom_complet IS NOT NULL THEN tx.nom_complet
ELSE ta.nom_complet
END AS nom_complet,
CASE
WHEN tx.cd_nom IS NOT NULL THEN tx.cd_nom
ELSE ta.cd_nom
END AS cd_nom,
CASE
WHEN tx.cd_ref IS NOT NULL THEN tx.cd_ref
ELSE ta.cd_ref
END AS cd_ref,
--s.cd_nom,
--t.cd_ref,
CASE
WHEN s.obs_null IS TRUE THEN 'oui'::text
ELSE 'non'::text
END::character(3) AS absence_observation,
s.effectif,
s.effectif_min,
s.effectif_max,
s.effectif_textuel,
CASE
WHEN s.regne ~~ 'Plantae'::text THEN s.type_effectif
ELSE NULL::text
END::saisie.enum_type_effectif AS strate_flore,
CASE
WHEN s.regne ~~ 'Plantae'::text THEN s.phenologie
ELSE NULL::text
END::saisie.enum_phenologie AS phenologie_flore,
CASE
WHEN s.regne ~~ 'Animalia'::text THEN s.type_effectif
ELSE NULL::text
END::saisie.enum_age AS age_faune,
CASE
WHEN s.regne ~~ 'Animalia'::text THEN s.phenologie
ELSE NULL::text
END::saisie.enum_sexe AS sexe_faune,
s.reprostatut AS reprostatut_faune,
s.determination,
s.remarque_obs AS rmq_observation,
md.liste_nom_auteur(s.observateur::character varying) AS observateurs,
md.liste_nom_auteur_structure(s.observateur::character varying) AS observateurs_v2,
num.nom_complet AS numerisateur,
md.liste_nom_structure(s.structure::character varying) AS structures,
CASE
WHEN s.diffusable IS TRUE THEN 'oui'::text
ELSE 'non'::text
END::character(3) AS diffusable,
s.statut_validation,
val.nom_complet AS validateur,
s.decision_validation,
s.code_insee AS insee_commune,
c.nom AS nom_commune,
l.nom AS lieu_dit,
s.elevation AS altitude_z,
s.longitude AS longitude_x,
s.latitude AS latitude_y,
s."precision",
s.localisation AS rmq_localisation,
s.id_origine,
s.geometrie::geometry(Point,2154) AS geom
FROM saisie.saisie_observation s
LEFT JOIN inpn.taxref t ON s.cd_nom = t.cd_nom
LEFT JOIN inpn.taxons_isere_absents_taxref ta ON s.cd_nom = ta.cd_nom::text
LEFT JOIN ref_inpn_taxref.taxref_v11 tx ON s.cd_nom = tx.cd_nom::text
JOIN md.etude e ON s.id_etude = e.id_etude
LEFT JOIN md.protocole p ON s.id_protocole = p.id_protocole
LEFT JOIN md.lot_donnee ld ON s.id_lot = ld.id_lot
LEFT JOIN observateurs num ON s.numerisateur = num.id_personne
LEFT JOIN observateurs val ON s.validateur = val.id_personne
LEFT JOIN ign_bd_topo.commune c ON s.code_insee = c.code_insee
LEFT JOIN ign_bd_topo.lieu_dit l ON s.id_lieu_dit = l.id
WITH DATA;
-- View indexes:
CREATE INDEX idx_vm_synthese_observations_date_debut_obs ON saisie.vm_synthese_observations USING btree (date_debut_obs);
CREATE INDEX idx_vm_synthese_observations_date_fin_obs ON saisie.vm_synthese_observations USING btree (date_fin_obs);
CREATE INDEX idx_vm_synthese_observations_date_obs ON saisie.vm_synthese_observations USING btree (date_obs);
CREATE INDEX idx_vm_synthese_observations_group1_inpn ON saisie.vm_synthese_observations USING btree (group1_inpn);
CREATE INDEX idx_vm_synthese_observations_group2_inpn ON saisie.vm_synthese_observations USING btree (group2_inpn);
CREATE UNIQUE INDEX idx_vm_synthese_observations_id_obs ON saisie.vm_synthese_observations USING btree (id_obs);
CREATE INDEX idx_vm_synthese_observations_nom_commune ON saisie.vm_synthese_observations USING btree (nom_commune);
CREATE INDEX idx_vm_synthese_observations_nom_latin ON saisie.vm_synthese_observations USING btree (nom_latin);
CREATE INDEX idx_vm_synthese_observations_observateurs ON saisie.vm_synthese_observations USING btree (observateurs);
CREATE INDEX idx_vm_synthese_observations_structures ON saisie.vm_synthese_observations USING btree (structures);
CREATE INDEX sidx_vm_synthese_observations_geom ON saisie.vm_synthese_observations USING gist (geom);
COMMENT ON MATERIALIZED VIEW saisie.vm_synthese_observations IS 'Vue matérialisée de synthèse des données d''observations issuent de la table "saisie.saise_observation" et reformattée pour faciliter son utilisation dans QGIS. La vue est rafraîchie toutes les heures avec un script PSQL executer en SHELL via une tâche planifiée CRON côté serveur. Cette vue matérialisée peut également être rafraîchi manuellement avec la commande : REFRESH MATERIALIZED VIEW CONCURRENTLY';
"""
grant_vm_synthese = """
-- Permissions
ALTER TABLE saisie.vm_synthese_observations OWNER TO admin_ra;
GRANT ALL ON TABLE saisie.vm_synthese_observations TO admin_ra;
GRANT SELECT ON TABLE saisie.vm_synthese_observations TO grp_consult;
GRANT ALL ON TABLE saisie.vm_synthese_observations TO grp_admin;
GRANT SELECT ON TABLE saisie.vm_synthese_observations TO sicen2_gr_consult;
"""
with con_sicen.begin() as cnx:
cnx.execute(drop_vm_synthese)
cnx.execute(vm_synthese_observations)
cnx.execute(grant_vm_synthese)
1_SICEN/v_synthese[pole].py
+250
View File
@@ -0,0 +1,250 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
import geopandas as gpd
from pycen import con_sicen
drop_v_inv = "DROP VIEW IF EXISTS saisie.v_synthese_invertebre"
v_synthese_invertebre = """
CREATE OR REPLACE VIEW saisie.v_synthese_invertebre
AS WITH update_data AS (
SELECT
vm.id_obs,
MAX(s.date_operation) date_update
FROM saisie.vm_synthese_observations vm
LEFT JOIN saisie.suivi_saisie_observation s USING (id_obs)
GROUP BY vm.id_obs
)
SELECT
vm.id_obs,
vm.etude,
vm.protocole,
vm.lot_donnee,
s.date_operation,
ud.date_update,
vm.date_obs::date,
vm.date_debut_obs::date,
vm.date_fin_obs::date,
vm.heure_obs::time,
vm.regne,
vm.classe,
vm.ordre,
vm.famille,
vm.group1_inpn,
vm.group2_inpn,
vm.nom_latin,
vm.nom_vern,
vm.nom_complet,
vm.cd_nom,
vm.cd_ref,
vm.absence_observation,
vm.effectif,
vm.effectif_min,
vm.effectif_max,
vm.effectif_textuel,
vm.strate_flore,
vm.phenologie_flore,
vm.age_faune,
vm.sexe_faune,
vm.reprostatut_faune,
vm.determination,
vm.rmq_observation,
vm.observateurs,
vm.observateurs_v2,
vm.numerisateur,
vm.structures,
vm.diffusable,
vm.statut_validation,
vm.validateur,
vm.decision_validation,
vm.insee_commune,
vm.nom_commune,
vm.lieu_dit,
vm.altitude_z,
vm.longitude_x,
vm.latitude_y,
vm."precision",
vm.rmq_localisation,
vm.geom,
so.id_origine
FROM saisie.vm_synthese_observations vm
LEFT JOIN ref_inpn_taxref.taxref_v11 tax ON vm.cd_nom::integer = tax.cd_nom
JOIN saisie.saisie_observation so USING (id_obs)
LEFT JOIN update_data ud USING (id_obs)
LEFT JOIN (SELECT * FROM saisie.suivi_saisie_observation WHERE operation = 'INSERT'::text) s USING (id_obs)
WHERE tax.regne = 'Animalia'::text AND tax.phylum <> 'Chordata'::text OR (vm.cd_nom::text = ANY (ARRAY['9999024'::text, '9999025'::text, '9999032'::text, '9999055'::text, '9999061'::text, '9999062'::text, '9999069'::text, '9999036'::text, '9999040'::text, '9999072'::text, '9999045'::text, '9999047'::text, '9999051'::text, '9999052'::text, '9999053'::text, '9999081'::text, '9999026'::text, '9999050'::text, '9999048'::text, '9999037'::text, '9999066'::text, '9999065'::text, '9999080'::text]));
"""
grant_v_inv = """
ALTER TABLE saisie.v_synthese_invertebre OWNER TO cen_admin;
GRANT ALL ON TABLE saisie.v_synthese_invertebre TO cen_admin;
GRANT SELECT ON TABLE saisie.v_synthese_invertebre TO grp_consult;
GRANT ALL ON TABLE saisie.v_synthese_invertebre TO grp_admin;
"""
with con_sicen.begin() as cnx:
cnx.execute(drop_v_inv)
cnx.execute(v_synthese_invertebre)
cnx.execute(grant_v_inv)
drop_v_vert = "DROP VIEW IF EXISTS saisie.v_synthese_vertebre"
v_synthese_vertebre = """
CREATE OR REPLACE VIEW saisie.v_synthese_vertebre
AS WITH update_data AS (
SELECT
vm.id_obs,
MAX(s.date_operation) date_update
FROM saisie.vm_synthese_observations vm
LEFT JOIN saisie.suivi_saisie_observation s USING (id_obs)
GROUP BY vm.id_obs
)
SELECT
vm.id_obs,
vm.etude,
vm.protocole,
vm.lot_donnee,
s.date_operation,
ud.date_update,
vm.date_obs::date,
vm.date_debut_obs::date,
vm.date_fin_obs::date,
vm.heure_obs::time,
vm.regne,
vm.classe,
vm.ordre,
vm.famille,
vm.group1_inpn,
vm.group2_inpn,
vm.nom_latin,
vm.nom_vern,
vm.nom_complet,
vm.cd_nom,
vm.cd_ref,
vm.absence_observation,
vm.effectif,
vm.effectif_min,
vm.effectif_max,
vm.effectif_textuel,
vm.strate_flore,
vm.phenologie_flore,
vm.age_faune,
vm.sexe_faune,
vm.reprostatut_faune,
vm.determination,
vm.rmq_observation,
vm.observateurs,
vm.observateurs_v2,
vm.numerisateur,
vm.structures,
vm.diffusable,
vm.statut_validation,
vm.validateur,
vm.decision_validation,
vm.insee_commune,
vm.nom_commune,
vm.lieu_dit,
vm.altitude_z,
vm.longitude_x,
vm.latitude_y,
vm."precision",
vm.rmq_localisation,
vm.geom,
so.id_origine
FROM saisie.vm_synthese_observations vm
LEFT JOIN ref_inpn_taxref.taxref_v11 tax ON vm.cd_nom::integer = tax.cd_nom
JOIN saisie.saisie_observation so USING (id_obs)
LEFT JOIN update_data ud USING (id_obs)
LEFT JOIN (SELECT * FROM saisie.suivi_saisie_observation WHERE operation = 'INSERT'::text) s USING (id_obs)
WHERE tax.regne = 'Animalia'::text AND tax.phylum = 'Chordata'::text OR (vm.cd_nom::text = ANY (ARRAY['9999042'::text, '9999028'::text, '9999029'::text, '9999056'::text, '9999058'::text, '9999067'::text, '9999068'::text, '9999030'::text, '9999031'::text, '9999034'::text, '9999034'::text, '9999035'::text, '9999035'::text, '9999038'::text, '9999039'::text, '9999070'::text, '9999073'::text, '9999057'::text, '9999054'::text, '9999049'::text, '9999022'::text, '9999027'::text, '9999043'::text, '9999044'::text, '9999046'::text, '9999041'::text, '9999033'::text, '9999071'::text, '9999064'::text, '9999063'::text, '9999060'::text, '9999059'::text, '9999074'::text, '9999023'::text, '9999082'::text, '9999083'::text]));
"""
grant_v_vert = """
ALTER TABLE saisie.v_synthese_vertebre OWNER TO cen_admin;
GRANT ALL ON TABLE saisie.v_synthese_vertebre TO cen_admin;
GRANT SELECT ON TABLE saisie.v_synthese_vertebre TO grp_consult;
GRANT ALL ON TABLE saisie.v_synthese_vertebre TO grp_admin;
"""
with con_sicen.begin() as cnx:
cnx.execute(drop_v_vert)
cnx.execute(v_synthese_vertebre)
cnx.execute(grant_v_vert)
drop_v_flo = "DROP VIEW IF EXISTS saisie.v_synthese_flore"
v_synthese_flore = """
CREATE OR REPLACE VIEW saisie.v_synthese_flore
AS WITH update_data AS (
SELECT
vm.id_obs,
MAX(s.date_operation) date_update
FROM saisie.vm_synthese_observations vm
LEFT JOIN saisie.suivi_saisie_observation s USING (id_obs)
GROUP BY vm.id_obs
)
SELECT
vm.id_obs,
vm.etude,
vm.protocole,
vm.lot_donnee,
s.date_operation,
ud.date_update,
vm.date_obs::date,
vm.date_debut_obs::date,
vm.date_fin_obs::date,
vm.heure_obs::time,
vm.regne,
vm.classe,
vm.ordre,
vm.famille,
vm.group1_inpn,
vm.group2_inpn,
vm.nom_latin,
vm.nom_vern,
vm.nom_complet,
vm.cd_nom,
vm.cd_ref,
vm.absence_observation,
vm.effectif,
vm.effectif_min,
vm.effectif_max,
vm.effectif_textuel,
vm.strate_flore,
vm.phenologie_flore,
vm.age_faune,
vm.sexe_faune,
vm.reprostatut_faune,
vm.determination,
vm.rmq_observation,
vm.observateurs,
vm.observateurs_v2,
vm.numerisateur,
vm.structures,
vm.diffusable,
vm.statut_validation,
vm.validateur,
vm.decision_validation,
vm.insee_commune,
vm.nom_commune,
vm.lieu_dit,
vm.altitude_z,
vm.longitude_x,
vm.latitude_y,
vm."precision",
vm.rmq_localisation,
vm.geom,
so.id_origine
FROM saisie.vm_synthese_observations vm
LEFT JOIN ref_inpn_taxref.taxref_v11 tax ON vm.cd_nom::integer = tax.cd_nom
JOIN saisie.saisie_observation so USING (id_obs)
LEFT JOIN update_data ud USING (id_obs)
LEFT JOIN (SELECT * FROM saisie.suivi_saisie_observation WHERE operation = 'INSERT'::text) s USING (id_obs)
WHERE tax.regne <> 'Animalia'::text OR (vm.cd_nom::text = ANY (ARRAY['9999008'::text, '9999003'::text, '9999005'::text, '9999006'::text, '9999004'::text, '9999009'::text, '9999010'::text, '9999011'::text, '9999013'::text, '9999014'::text, '9999015'::text, '9999016'::text, '9999017'::text, '9999018'::text, '9999019'::text, '9999020'::text, '9999021'::text, '9999007'::text, '9999012'::text, '9999001'::text, '9999002'::text]));
"""
grant_v_flo = """
ALTER TABLE saisie.v_synthese_flore OWNER TO cen_admin;
GRANT ALL ON TABLE saisie.v_synthese_flore TO cen_admin;
GRANT SELECT ON TABLE saisie.v_synthese_flore TO grp_consult;
GRANT ALL ON TABLE saisie.v_synthese_flore TO grp_admin;
"""
with con_sicen.begin() as cnx:
cnx.execute(drop_v_flo)
cnx.execute(v_synthese_flore)
cnx.execute(grant_v_flo)
2_MEDWET/0_BROUILLONS/medwet.py
+87
View File
@@ -0,0 +1,87 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Nom : : medwet.py
#Description :
#Copyright : 2021, CEN38
#Auteur : Colas Geier
#Version : 1.0
import pandas as pd
import numpy as np
from pycen import bdd
from sqlalchemy import create_engine
# Parametres bdd
user = 'cen_admin'
pwd = '#CEN38@venir'
adr = '192.168.0.3'
base = 'bd_cen38'
schema = 'zh'
table = 'cr_cen38_zh_medwet_v2021'
dict_col = [{'criete_delimit': 'critere_delim'}]
con = create_engine('postgresql+psycopg2://{0}:{1}@{2}/{3}'.format(user,pwd,adr,'bd_cen_new'), echo=False)
# Connexion bdd
bd = bdd.CEN(
user = user,
pwd = pwd,
adr = adr,
base = base
# schema = schema
)
df = bd.get_table(
schema = schema,
table = table)
##### critere_delimitation
critere_delimitation = pd.DataFrame(df.criete_delimit)
# remplacement d'une chaine de caractère
critere_delimitation.criete_delimit = critere_delimitation.criete_delimit.str.replace('critère de délimitation ZH : ', '')
# elimination des lignes vides
critere_delimitation.dropna(inplace = True)
# split des champs regroupant plusieurs infos et concaténation
critere_delimitation.criete_delimit = pd.concat(
[pd.Series(row['criete_delimit'].split(' // ')) for _, row in critere_delimitation.iterrows()]
).reset_index(drop=True)
liste_critere_delim = pd.DataFrame(critere_delimitation.criete_delimit.unique(), columns=['caracteristique'])
liste_critere_delim = liste_critere_delim.append(pd.DataFrame(['tutu'], columns=['caracteristique']))
liste_critere_delim.reset_index(inplace=True, drop=True)
liste_critere_delim.to_sql(name='zh_delimitation', con=con, schema='zone_humide', index=False, if_exists='append')
##### regime_hydrique_entree
regime_hydrique_entree = pd.DataFrame(df.regime_hydrique_entree)
regime_hydrique_entree.dropna(inplace = True)
regime_hydrique_entree.regime_hydrique_entree = pd.concat(
[pd.Series(row['regime_hydrique_entree'].split(' // ')) for _, row in regime_hydrique_entree.iterrows()]
).reset_index(drop=True)
regime_hydrique_entree.reset_index(drop=True, inplace=True)
regime_hydrique_entree[['ecoulement_entree', 'toponymie', 'temporalite']] = regime_hydrique_entree.regime_hydrique_entree.str.split(' ; ', 2, expand=True)
# regime_hydrique_entree[['toponymie', 'temporalite']] = regime_hydrique_entree.temporalite.str.split(' ; ', 1, expand=True)
regime_hydrique_entree.drop(columns='regime_hydrique_entree', inplace = True)
regime_hydrique_entree.ecoulement_entree = regime_hydrique_entree.ecoulement_entree.str.replace('Entrée d\'eau : ', '')
regime_hydrique_entree.toponymie = regime_hydrique_entree.toponymie.str.replace('Toponymie : ', '')
regime_hydrique_entree.temporalite = regime_hydrique_entree.temporalite.str.replace('Permanence : ', '')
regime_hydrique_entree.ecoulement_entree.unique()
regime_hydrique_entree.toponymie.unique()
regime_hydrique_entree.temporalite.unique()
##### regime_hydrique_sortie
regime_hydrique_sortie = pd.DataFrame(df.regime_hydrique_sortie)
regime_hydrique_entree.dropna(inplace = True)
regime_hydrique_sortie.regime_hydrique_sortie = pd.concat(
[pd.Series(row['regime_hydrique_sortie'].split(' // ')) for _, row in regime_hydrique_sortie.iterrows()]
).reset_index(drop=True)
regime_hydrique_sortie['ecoulement'], regime_hydrique_sortie['toponymie'], regime_hydrique_sortie['permanence']
regime_hydrique_freq = pd.DataFrame(df.regime_hydrique_freq)
regime_hydrique_origine = pd.DataFrame(df.regime_hydrique_orig)
regime_hydrique_etendue = pd.DataFrame(df.regime_hydrique_etendue)
2_MEDWET/0_BROUILLONS/recup_zh.py
+2256
View File
File diff suppressed because it is too large Load Diff
2_MEDWET/get_medwet.py
+71
View File
@@ -0,0 +1,71 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Nom : : get_medwet.py
#Description :
#Copyright : 2021, CEN38
#Auteur : Colas Geier
#Version : 1.0
import pymedwet as pym
import pandas as pd
if __name__ == "__main__":
NAME_OUT = '~/Documents/9_PROJETS/1_ZH/inventaire_zh_2021.xlsx'
lst_fct = {
'Infos générales': pym.get_SiteInfo,
'Corine Biotope' : pym.get_cb,
'Délimitation de la zh': pym.get_Delim_espaceFct,
'Description de la zh' : pym.get_descrp_zh,
'Fonctmt de la zh': pym.get_fctmt_zh,
'Fonctions de la zh' : pym.get_fct_zh,
'Evaluation de la zh': pym.get_eval_glob,
'Observations' : pym.get_obs,
'Bibliographie': pym.get_biblio
}
print('INIT récupération des données ...')
df = {}
for f,fonction in enumerate(lst_fct):
name = fonction
fct = lst_fct[fonction]
df1 = fct(pym.db_file1)
df2 = fct(pym.db_file2)
if f == 0:
lst_stcd = df1[df1.SITE_COD.isin(df2.SITE_COD)].SITE_COD
if fct == pym.get_SiteInfo:
df2.loc[df2.SITE_COD.isin(lst_stcd), 'OTHER_NAME'] = df1.loc[df1.SITE_COD.isin(lst_stcd),'SITE_NAME'].tolist()
if isinstance(df1, pd.DataFrame) and isinstance(df2, pd.DataFrame):
df[f] = pd.concat([df1[~df1.SITE_COD.isin(lst_stcd)], df2])
df[f].name = name
elif isinstance(df1, dict) and isinstance(df2, dict):
df[f] = {}
df[f]['title'] = name
for d in df1:
df[f][d] = pd.concat([df1[d][~df1[d].SITE_COD.isin(lst_stcd)], df2[d]])
print('INIT écriture du fichier ...')
# Ecriture des données
with pd.ExcelWriter(NAME_OUT) as writer:
for d in df:
DF = df[d]
if isinstance(DF, pd.DataFrame):
DF.to_excel(writer,sheet_name=DF.name,startrow=1 , startcol=0, index=False, header=DF.columns)
ws = writer.book.active
writer.sheets[DF.name].cell(1,1,value=DF.name)
writer.save()
elif isinstance(DF, dict):
for i,d in enumerate(DF):
if d == 'title':
continue
if i == 1:
row = 1
col = 0
else:
col = DF[d].shape[1] + col + 3
DF[d].to_excel(writer,sheet_name=DF['title'],startrow=row , startcol=col, index=False)
ws = writer.book.active
writer.sheets[DF['title']].cell(column=col+1,row=row,value=d)
writer.save()
import sys
sys.exit('END SCRIPT ...')
2_MEDWET/get_zh_cen.py
+511
View File
@@ -0,0 +1,511 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Nom : : get_zh_cen.py
#Description :
#Copyright : 2021, CEN38
#Auteur : Colas Geier
#Version : 1.0
import re
import pandas as pd
import pandas_access as mdb
import numpy as np
from sqlalchemy.sql.expression import column
from sqlalchemy import create_engine
from geoalchemy2 import Geometry
isin_bdd = True
# Parametres bdd OUT
user_zh = 'postgres'
pwd_zh = 'tutu'
adr_zh = '192.168.60.10'
base_zh = 'bd_cen'
con_zh = create_engine('postgresql+psycopg2://{0}:{1}@{2}/{3}'.format(user_zh,pwd_zh,adr_zh,base_zh), echo=False)
#####################################
### Fonctions générales ###
#####################################
def _aggr_cols(df, lst_col, sep=''):
df['aggreg'] = ''
for c,col in enumerate(lst_col):
add = ''
if c > 0:
add = sep
df.loc[~df[col].isna(),'aggreg'] = df.loc[~df[col].isna(),'aggreg'] + add + df.loc[~df[col].isna(),col]
return df
def to_tuple(obj):
if isinstance(obj, list): obj = tuple(obj)
if isinstance(obj, (int, str)) : obj = tuple([obj])
return obj
def to_colStringSQL(obj):
if isinstance(obj, (int, str)) : obj = str(obj)
if isinstance(obj, list): obj = ",".join(obj)
return obj
def to_upper(obj):
if isinstance(obj, tuple): obj = tuple([o.upper() for o in list(obj)])
if isinstance(obj, list) : obj = [o.upper() for o in obj]
if isinstance(obj, str) : obj = obj.upper()
return obj
def to_upperfirst(obj):
if isinstance(obj, tuple): obj = tuple([o.upper()[0] + o.lower()[1:] for o in list(obj)])
if isinstance(obj, list) : obj = [o.upper()[0] + o.lower()[1:] for o in obj]
if isinstance(obj, str) : obj = obj.upper()[0] + obj.lower()[1:]
return obj
def _get_table(con, schema, table, ids=None, nom=None, cols=None, params_col={}):
sql = 'SELECT * FROM {sch}.{tab}'.format(sch=schema, tab=table)
if cols : sql = sql.replace('*', to_colStringSQL(cols) )
if ids or nom or params_col : sql = sql + ' WHERE '
if ids : sql = sql + 'id IN %(ids)s'
if ids and (nom or params_col) : sql = sql + ' AND '
if nom : sql = sql + 'nom IN %(nom)s'
if nom and params_col : sql = sql + ' AND '
if params_col :
sql = sql + ' AND '.join([k + ' IN %({})s'.format(k) for k in params_col.keys()])
params_col = {key:to_tuple(params_col[key]) for key in params_col.keys()}
df = pd.read_sql(
sql = sql,
con = con,
params = {'ids': to_tuple(ids), 'nom': to_tuple(nom), **params_col })
return df
#####################################
### schema personnes ###
#####################################
class pers:
def __init__(self):
self.schema = 'personnes'
self.con = con_zh
# self._get_table = _get_table
def get_auteur(self, nom=None, prenom=None):
sql = 'SELECT * FROM %s.personne'%self.schema
if nom or prenom : sql = sql + ' WHERE '
if nom :
sql = sql + 'nom IN %(nom)s'
nom = to_upper(nom)
if nom and prenom : sql = sql + ' AND '
if prenom :
sql = sql + 'prenom IN %(prenom)s'
prenom = to_upperfirst(prenom)
df = pd.read_sql(
sql = sql,
con = self.con,
params = {'nom': to_tuple(nom), 'prenom': to_tuple(prenom) })
return df
def get_organisme(self, ids=None, nom=None):
table = 'organisme'
return _get_table(self.con, self.schema, table, ids=ids, nom=nom)
#####################################
### schema sites ###
#####################################
class sites:
def __init__(self):
self.schema = 'sites'
self.con = con_zh
self.typ_milieux = self._get_typ_milieux()
self.typo_sdage = self._get_typo_sdage()
self.typ_site = self._get_typ_site()
self.auteur = pers().get_auteur()
self.organisme = pers().get_organisme()
def _get_typ_milieux(self, ids=None, nom=None):
table = 'type_milieu'
df = _get_table(self.con, self.schema, table, ids=ids, nom=nom)
return df
def _get_typo_sdage(self, ids=None, nom=None):
table = 'typo_sdage'
df = _get_table(self.con, self.schema, table, ids=ids, nom=nom)
return df
def _get_typ_site(self, ids=None, nom=None):
table = 'type_site'
df = _get_table(self.con, self.schema, table, ids=ids, nom=nom)
return df
def _merge_orga(self, df, split_cols):
org = self.organisme
aut = self.auteur
df = df.copy()
for c in split_cols:
if not isinstance(df[c], int): df[c] = df[c].astype(float)
df[c].replace(aut.id.tolist(), aut.id_organisme.tolist(), inplace=True)
df[c].replace(org.id.tolist(), org.nom.tolist(), inplace=True)
df['organisme'] = None
for c in split_cols:
df.loc[df.organisme.isna(), 'organisme'] = df.loc[df['organisme'].isna(), c]
for c in split_cols:
comp = df.loc[~df[c].isna(),'organisme'].compare(df.loc[~df[c].isna(), c])
if not comp.empty:
comp['test'] = comp.apply(lambda x: x['other'] in x['self'], axis=1)
comp = comp[~comp.test]
if not comp.empty:
df.loc[comp.index,'organisme'] = comp.self + ' & ' + comp.other
df.drop(columns=split_cols, inplace=True)
return df
def _merge_author(self, df, col_aut, orga=False):
# récupération des auteurs
aut = self.auteur.fillna('')
aut['nom_prenom'] = (aut['nom'] + ' ' + aut['prenom']).str.strip()
aut['id'] = aut['id'].astype(str)
# merge des auteurs
r_id = df[['id', col_aut]].copy()
r_idSplit = r_id[col_aut].str.split(' & ', expand=True)
r_id = r_id.join(r_idSplit)
cSplit = r_idSplit.shape[1]
cSplit = list(range(cSplit))
if orga:
# récup des organismes
org = self._merge_orga(r_id, cSplit)
r_id[cSplit] = r_id[cSplit].replace(aut['id'].tolist(),aut['nom_prenom'].tolist())
r_id = _aggr_cols(r_id,cSplit,' & ') \
.rename(columns={'aggreg': 'auteur'}) \
.drop(columns=cSplit)
if orga:
# merge des organismes
r_id = pd.merge(r_id,org, on=['id', col_aut])
df = pd.merge(df,r_id, on=['id', col_aut]) \
.drop(columns=[col_aut])
return df
def get_sitesInfos(self, ids=None, nom=None, columns=None, with_nameOrga=False, details=False):
drop = []
table = 'sites'
df = _get_table(self.con, self.schema, table, ids=ids, nom=nom, cols=columns)
# récupération des auteurs
if 'id_auteur' in df.columns:
df = self._merge_author(df=df, col_aut='id_auteur', orga=with_nameOrga)
# merge type_site
if 'id_type_site' in df.columns:
df = pd.merge(df, self.typ_site, left_on='id_type_site', right_on='id', suffixes=('','_y') ) \
.drop(columns=['id_type_site', 'id_y']) \
.rename(columns={'nom_y': 'type_site', 'description': 'desc_type_site'})
drop += ['desc_type_site']
# merge typo_sdage
if 'id_typo_sdage' in df.columns:
df = pd.merge(df, self.typo_sdage, left_on='id_typo_sdage', right_on='id', suffixes=('','_y') ) \
.drop(columns=['id_typo_sdage', 'id_y']) \
.rename(columns={'nom_y': 'typo_sdage', 'description': 'desc_typo_sdage'})
drop += ['desc_typo_sdage']
# merge type_milieu
if 'id_type_milieu' in df.columns:
df = pd.merge(df, self.typ_milieux, left_on='id_type_milieu', right_on='id', suffixes=('','_y') ) \
.drop(columns=['id_type_milieu', 'id_y']) \
.rename(columns={'nom_y': 'type_milieu', 'description': 'desc_type_milieu', 'nom_court': 'nom_court_milieu'})
drop += ['desc_type_milieu', 'nom_court_milieu']
if not details:
df.drop(columns=drop, inplace=True)
return df.sort_values('id')
def get_sitesGeom(self, id_site=None, nom_site=None, columns=None, last_update=False, with_nameOrga=False):
from shapely.wkb import loads
import geopandas as gpd # set_geometry
if columns:
if not isinstance(columns, list): columns = [columns]
if 'id' not in columns: columns.insert(0,'id')
if 'id_site' not in columns: columns.insert(1,'id_site')
if 'geom' not in columns: columns.insert(2,'geom')
table = 'sites'
df = _get_table(self.con, self.schema, table, ids=id_site, nom=nom_site, cols='id')
idSite = df.id.tolist()
table = 'r_sites_geom'
df = _get_table(self.con, self.schema, table, params_col={'id_site':idSite}, cols=columns)
if last_update:
df.drop_duplicates(subset=['id_site'], keep='last', inplace=True)
df.reset_index(inplace=True, drop=True)
df['geom'] = [(loads(geom, hex=True)) for geom in df['geom']]
df = df.set_geometry('geom', crs='EPSG:2154')
# merge auteur
if 'id_auteur' in df.columns:
df = self._merge_author(df=df, col_aut='id_auteur', orga=with_nameOrga)
return df
#####################################
### schema sites ###
#####################################
class zh:
def __init__(self):
self.schema = 'zones_humides'
self.con = con_zh
def _get_param(self, param_table, type_table=None, type_court=True):
if type_table:
typ = _get_table(self.con, self.schema, table=type_table)
par = _get_table(self.con, self.schema, table=param_table, params_col={'id_type':typ.id.tolist()})
df = pd.merge(par, typ, left_on='id_type', right_on='id', how='left', suffixes=(None, '_typ')) \
.drop(columns=['id_type','id_typ'])
if 'description_typ' in df.columns: del df['description_typ']
if type_court: df = df.drop(columns=['nom_typ']).rename(columns={'nom_court_typ':'type'})
else : df = df.drop(columns=['nom_court_typ'],errors='ignore').rename(columns={'nom_typ':'type'})
df = df.set_index(['id', 'type']).reset_index()
else:
df = _get_table(self.con, self.schema, table=param_table)
return df
def _get_relation_tab(self, tab, id_site=None, nom_site=None, last_update=False, geom=False):
table = 'sites'
dfSG = sites().get_sitesGeom(columns='date', id_site=id_site, nom_site=nom_site, last_update=last_update)
if not geom and not dfSG.empty:
dfSG.drop('geom',1,inplace=True)
ids = dfSG.id.tolist()
table = tab
if ids :
df = _get_table(self.con, self.schema, table, params_col={'id_geom_site':ids})
if not df.empty:
df = pd.merge(dfSG,df, left_on='id', right_on='id_geom_site', suffixes=('_x', None)) \
.drop(['id_x','id_geom_site'],1) \
.set_index('id').reset_index()
return df
else:
print('PAS de géometries de sites sélectionnées ...')
def get_delim(self, id_site=None, nom_site=None, last_update=False, geom=False, nom_type_court=True):
table = 'r_site_critdelim'
df = self._get_relation_tab(tab=table,id_site=id_site,nom_site=nom_site,last_update=last_update,geom=geom)
dic = self._get_param(type_table='type_param_delim_fct', param_table='param_delim_fct', type_court=nom_type_court)
if not df.empty:
df = pd.merge(df,dic, left_on='id_crit_delim', right_on='id', suffixes=(None,'_y')) \
.drop(['id_y','id_crit_delim'],1) \
.rename(columns={'description_y':'desc_param', 'nom_court':'nom_court_crit','nom':'nom_crit'}) \
.sort_values('id_site')
if df.nom_court_crit.isnull().sum() == df.shape[0] : del df['nom_court_crit']
# typ = df.type.unique()
# x = {}
# for t in typ:
# x[t] = df[df.type == t]
# x[t] = x[t].rename(columns={'nom': t}) \
# .reset_index(drop=True)
return df
def get_fct(self, id_site=None, nom_site=None, last_update=False, geom=False, nom_type_court=True):
table = 'r_site_fctecosociopatri'
df = self._get_relation_tab(tab=table,id_site=id_site,nom_site=nom_site,last_update=last_update,geom=geom)
dic = self._get_param(type_table='type_param_fct', param_table='param_fct_eco_socio_patri', type_court=nom_type_court)
if not df.empty:
df = pd.merge(df,dic, left_on='id_fct', right_on='id', suffixes=(None,'_y')) \
.drop(['id_y','id_fct'],1) \
.rename(columns={'description_y':'desc_param', 'nom_court':'nom_court_fct','nom':'nom_fct'}) \
.sort_values('id_site')
if df.nom_court_fct.isnull().sum() == df.shape[0] : del df['nom_court_fct']
return df
def get_connex(self, id_site=None, nom_site=None, last_update=False, geom=False):
table = 'r_site_type_connect'
df = self._get_relation_tab(tab=table,id_site=id_site,nom_site=nom_site,last_update=last_update,geom=geom)
dic = self._get_param(param_table='param_type_connect')
if not df.empty:
df = pd.merge(df,dic, left_on='id_param_connect', right_on='id', suffixes=(None,'_y')) \
.drop(['id_y','id_param_connect'],1) \
.rename(columns={'description_y':'desc_param', 'nom':'connexion'}) \
.sort_values('id_site')
return df
def get_sub(self, id_site=None, nom_site=None, last_update=False, geom=False):
table = 'r_site_sub'
df = self._get_relation_tab(tab=table,id_site=id_site,nom_site=nom_site,last_update=last_update,geom=geom)
dic = self._get_param(type_table='type_param_sub', param_table='param_sub', type_court=False)
d1 = dic[dic.type == 'Submersion étendue']
d2 = dic[dic.type == 'Submersion fréquente']
if not df.empty:
df = pd.merge(df,d1, how='left', left_on='id_etendsub', right_on='id', suffixes=(None,'_y')) \
.drop(['id_y','id_etendsub', 'type'],1) \
.rename(columns={'description':'desc_param_etend', 'nom':'Submersion étendue'})
df = pd.merge(df,d2, how='left', left_on='id_freqsub', right_on='id', suffixes=(None,'_y')) \
.drop(['id_y','id_freqsub', 'type'],1) \
.rename(columns={'description':'desc_param_freq', 'nom':'Submersion fréquente'}) \
.sort_values('id_site')
df.rename(columns={'id_origsub': 'origine_sub'}, inplace=True)
if df['desc_param_etend'].isnull().sum() == df.shape[0] : del df['desc_param_etend']
if df['desc_param_freq'].isnull().sum() == df.shape[0] : del df['desc_param_freq']
return df
def get_usageprocess(self, id_site=None, nom_site=None, last_update=False, geom=False):
table = 'r_site_usageprocess'
df = self._get_relation_tab(tab=table,id_site=id_site,nom_site=nom_site,last_update=last_update,geom=geom)
dic1 = self._get_param(param_table='param_usageprocess')
dic2 = self._get_param(param_table='param_position')
if not df.empty:
df = pd.merge(df,dic1, how='left', left_on='id_usageprocess', right_on='id', suffixes=(None,'_y')) \
.drop(['id_y','id_usageprocess'],1) \
.rename(columns={'description':'desc_param_usag', 'nom':'usageprocess'})
df = pd.merge(df,dic2, how='left', left_on='id_position', right_on='id', suffixes=(None,'_y')) \
.drop(['id_y','id_position'],1) \
.rename(columns={'description':'desc_param_pos', 'nom':'position'}) \
.sort_values('id_site')
return df
def _get_r_toponymie(self, ids=None):
table = 'r_toponymie'
df = _get_table(self.con, self.schema, table=table, ids=ids)
dic1 = self._get_param(param_table='liste_table_topohydro')
if not df.empty:
df = pd.merge(df,dic1, left_on='id_orig', right_on='id', suffixes=(None,'_y')) \
.drop(['id_y','id_orig'],1)
n_tab = df.nom_table.unique()
for tab in n_tab:
iids = df.loc[df.nom_table == tab, 'id_topo'].to_list()
if tab == 'orig_hydro': dic = _get_table(self.con, self.schema, table='orig_hydro', ids=iids)
if tab == 'troncon_hydro': dic = ref_hydro.get_troncon(cols=['id','nom'], ids=iids)
df.loc[df.nom_table == tab, 'id_topo'] = df.loc[df.nom_table == tab, 'id_topo'].replace(dic.id.to_list(),dic.nom.to_list())
if tab == 'troncon_hydro': df = pd.merge(df, dic, left_on='id_topo', right_on='nom', suffixes=(None,'_y')) \
.drop(columns=['id_y', 'nom'])
df.rename(columns={'id_topo':'toponymie'})
return df
def get_regHydro(self, id_site=None, nom_site=None, last_update=False, geom=False):
table = 'r_site_reghydro'
df = self._get_relation_tab(tab=table,id_site=id_site,nom_site=nom_site,last_update=last_update,geom=geom)
dic1 = self._get_param(param_table='param_reg_hydro')
dic2 = self._get_param(param_table='param_permanance')
if not df.empty:
dic3 = self._get_r_toponymie(ids=df.id_toponymie.unique().tolist())
df.in_out = df.in_out.replace([True,False],['entree','sortie'])
df = pd.merge(df,dic1, left_on='id_reg_hydro', right_on='id', suffixes=(None,'_y')) \
.drop(['id_y','id_reg_hydro'],1) \
.rename(columns={'description':'desc_param_regHydri', 'nom':'regime_hydri'})
df = pd.merge(df,dic2, left_on='id_permanance', right_on='id', suffixes=(None,'_y')) \
.drop(['id_y','id_permanance'],1) \
.rename(columns={'description':'desc_param_perm', 'nom':'permanance'})
df = pd.merge(df,dic3, left_on='id_toponymie', right_on='id', suffixes=(None,'_y')) \
.drop(['id_y','id_toponymie'],1) \
.rename(columns={'description':'desc_topo'}) \
.sort_values('id_site')
return df
def get_habitat(self, id_site=None, nom_site=None, last_update=False, geom=False):
table = 'r_site_habitat'
df = self._get_relation_tab(tab=table,id_site=id_site,nom_site=nom_site,last_update=last_update,geom=geom)
if not df.empty:
dic = ref_hab().get_CB(ids=df.id_cb.unique().tolist(),cols=['id','lb_hab_fr'])
df = pd.merge(df,dic, left_on='id_cb', right_on='id', suffixes=(None,'_y')) \
.drop(['id_y'],1) \
.rename(columns={'id_cb':'code_cb'}) \
.sort_values('id_site')
return df
#####################################
### schema ref_habitats ###
#####################################
class ref_hab:
def __init__(self):
self.schema = 'ref_habitats'
self.con = con_zh
def get_CB(self, ids=None, cols=None, params_col={}):
table = 'corine_biotope'
df = _get_table(self.con, self.schema, table=table, ids=ids, cols=cols, params_col=params_col)
return df
#####################################
### schema ref_hydro ###
#####################################
class ref_hydro:
def __init__(self):
self.schema = 'ref_hydro'
self.con = con_zh
def get_troncon(self, ids=None, cols=None, params_col={}):
table = 'troncon_hydro'
df = _get_table(self.con, self.schema, table=table, ids=ids, cols=cols, params_col=params_col)
return df
#####################################
### Bilan ###
#####################################
def get_bilan(code_site=None, nom_site=None):
'''
:sites: list,str. Nom de code du site de la zh.
'''
SITES = sites()
ZH = zh()
info = SITES.get_sitesInfos(ids=code_site, nom=nom_site)
CB = ZH.get_habitat(id_site=code_site, nom_site=nom_site)
delim = ZH.get_delim(id_site=code_site, nom_site=nom_site)
desc = ZH.get_usageprocess(id_site=code_site, nom_site=nom_site)
rghyd = ZH.get_regHydro(id_site=code_site, nom_site=nom_site)
subm = ZH.get_sub(id_site=code_site, nom_site=nom_site)
conn = ZH.get_connex(id_site=code_site, nom_site=nom_site)
fct = ZH.get_fct(id_site=code_site, nom_site=nom_site)
evall = SITES.get_sitesGeom().drop(columns=['geom'])
sub_con = pd.merge(subm, conn, how='outer', on=['id', 'id_site', 'date']) \
.rename(columns={'description': 'desc_connex'})
fctmt = {
'entree_eau': rghyd[rghyd.in_out == 'entree'].drop(columns=['in_out']),
'sortie_eau': rghyd[rghyd.in_out == 'sortie'].drop(columns=['in_out']),
'sub_connex': sub_con,
}
lst_df = {
'infos':info,
'corine_biotope': CB,
'delimitation': delim,
'description': desc,
'fonctionnement': fctmt,
'fonction': fct,
'evaluation': evall}
for key in lst_df:
if isinstance(lst_df[key], pd.DataFrame): lst_df[key].name = key
if isinstance(lst_df[key], dict):
for d in lst_df[key]:
lst_df[key][d].name = d
lst_df[key]['title'] = key
return lst_df
def write_bilan(df, output):
'''
:df: dict. Dictionnaire de DataFrame.
Ecriture d'un feuillet par élément du dictionnaire.
Le nom du DataFrame est le titre du feuillet.
output: str. chemin_vers_mon_fichier/mon_fichier.xlsx
'''
# Ecriture des données
with pd.ExcelWriter(output) as writer:
for d in df:
DF = df[d]
if isinstance(DF, pd.DataFrame):
DF.to_excel(writer,sheet_name=DF.name,startrow=1 , startcol=0, index=False, header=DF.columns)
ws = writer.book.active
writer.sheets[DF.name].cell(1,1,value=DF.name)
writer.save()
elif isinstance(DF, dict):
for i,d in enumerate(DF):
if d == 'title': continue
if i == 0:
row = 1
col = 0
else:
col = DF[d].shape[1] + col + 3
DF[d].to_excel(writer,sheet_name=DF['title'],startrow=row , startcol=col, index=False)
ws = writer.book.active
writer.sheets[DF['title']].cell(column=col+1,row=row,value=d)
writer.save()
2_MEDWET/recup_zh_from_medwet.py
+939
View File
@@ -0,0 +1,939 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Nom : : recup_zh_from_medwet.py
#Description :
#Copyright : 2021, CEN38
#Auteur : Colas Geier
#Version : 1.0
import re
import pandas as pd
import pandas_access as mdb
import numpy as np
from sqlalchemy import create_engine
from geoalchemy2 import Geometry
isin_bdd = True
# Parametres bdd OUT
user_zh = 'postgres'
pwd_zh = 'tutu'
adr_zh = '192.168.60.10'
base_zh = 'bd_cen'
con_zh = create_engine('postgresql+psycopg2://{0}:{1}@{2}/{3}'.format(user_zh,pwd_zh,adr_zh,base_zh), echo=False)
# Read MS access database
db_file1 = '/home/colas/Documents/5_BDD/ZONES_HUMIDES/MEDWET_v1.mdb'
db_file2 = '/home/colas/Documents/5_BDD/ZONES_HUMIDES/MEDWET_V2.mdb'
db_file74 = '/home/colas/Documents/13_CEN74/medwet2gn_ZH/bdd/Inventaires ZHRMCvivante.mdb'
# df_med1 = mdb.read_table(db_file1, "SITEINFO")
# df_med2 = mdb.read_table(db_file2, "SITEINFO")
#####################################
### Obsrevation de la zone humide ###
#####################################
def _aggr_cols(df, lst_col):
df['aggreg'] = None
for col in lst_col:
iscol = ~df[col].isna()
isagg = ~df['aggreg'].isna()
df.loc[iscol & isagg, 'aggreg'] = df.loc[iscol & isagg, 'aggreg'] + \
' ; ' + col + ' : ' + df.loc[iscol & isagg, col]
df.loc[iscol & ~isagg, 'aggreg'] = col + ' : ' + df.loc[iscol & ~isagg, col]
return df
def _del_na_col(df):
for col in df.columns:
tmp = pd.notna(df[col]).unique()
if len(tmp) == 1 and False in tmp:
del df[col]
return df
def __get_auteur__(db_file):
df = mdb.read_table(db_file, 'MWDKC')
df = _del_na_col(df)
df.drop(columns=['FAX', 'E_MAIL', 'COUNTRY', 'PHONE', 'ADDRESS', 'CITY'], inplace=True)
return df
def __get_DicGenREGLE__():
d = {'CODE':['1','2','3','4','5'], 'DESCR':[
'protection : nationale (PN) / régionale (PR)',
'livres rouges : nationale (LN) / régionale (LR)',
'Directives Habitats (annexe II ou annexe IV)',
'Directive Oiseaux (annexe I)',
'autres statuts réglementaires (Convention de Bonn, Convention de Bern…)',
]}
return pd.DataFrame(data=d)
def __get_DicEsp__(db_file, dic, detail=False):
df = mdb.read_table(db_file, dic)
df = _del_na_col(df)
df.drop(columns=[
'FRANCE', 'ITALY', 'GREECE', 'SPAIN', 'PORTUGAL',
'ANNEX_II', 'SPCLIV', 'SPCLV', 'SPBCAX1','IUCN_REDL'], inplace=True)
df_cols = df.columns
df_cols = df_cols[df_cols.str.startswith(('SPBONN','ANNEX'))]
if not detail and not df_cols.empty:
df.drop(columns=df_cols, inplace=True)
return df
def __get_ObsEsp__(db_file, dic):
df = mdb.read_table(db_file, dic)
df = _del_na_col(df)
return df
def __get_SiteInfo__(db_file):
df = mdb.read_table(db_file, 'SITEINFO')
df = _del_na_col(df)
df.drop(columns=[
'COMPLE_COD', 'AREA_EF', 'AREA_COMP'
], inplace=True)
for col in ['PROTOCOLE', 'VALEUR_HUMIDITE']:
if col in df.columns:
df.drop(columns=[col], inplace=True)
return df
def __insert_type_esp__(df, type_esp_name):
for col in df.columns:
if col == 'ORDER': break
fix_col = col
tmp = df.loc[:,:fix_col]
tmp['TYPE_ESP'] = type_esp_name
return tmp.join(df.loc[:,'ORDER':])
def _get_obs(db_file, tab_obs, dic_obs, detail=False):
df_obs = __get_ObsEsp__(db_file, tab_obs)
df_dic = __get_DicEsp__(db_file, dic_obs, detail)
df = pd.merge(df_obs,df_dic,how='left', left_on='SPEC_COD', right_on='ID_COD')
if tab_obs == 'OBSFLO':
df.rename(columns={'FLO_MOIS': 'BRIDDATE_M', 'FLO_YEAR': 'BRIDDATE_Y'}, inplace=True)
if not detail:
df.drop(columns=['SPEC_COD','ID_COD'], inplace=True)
return df
def get_obs(db_file, type_obs='all', detail=False):
'''
:db_file: str. Nom de la base de données access
:type_obs: str ou list. Code des observations à extraire de
la bdd access MEDWET : 'amphi', 'bird', 'flore', 'fish',
'invert', 'mamm', 'rept'. 'all' si tous.
:detail: bool.
'''
obs_amp = pd.DataFrame()
obs_brd = pd.DataFrame()
obs_flo = pd.DataFrame()
obs_fsh = pd.DataFrame()
obs_inv = pd.DataFrame()
obs_mam = pd.DataFrame()
obs_rep = pd.DataFrame()
if 'all' == type_obs or 'all' in type_obs:
type_obs = ['amphi', 'bird', 'flore', 'fish', 'invert', 'mamm', 'rept']
if 'amphi' == type_obs or 'amphi' in type_obs:
tab_obs = 'OBSAMP'
dic_obs = 'DICAMP'
obs_amp = _get_obs(db_file, tab_obs, dic_obs, detail)
obs_amp = __insert_type_esp__(obs_amp, 'Amphibien')
if 'bird' == type_obs or 'bird' in type_obs:
tab_obs = 'OBSBRD'
dic_obs = 'DICBRD'
obs_brd = _get_obs(db_file, tab_obs, dic_obs, detail)
obs_brd = __insert_type_esp__(obs_brd, 'Oiseau')
if 'flore' == type_obs or 'flore' in type_obs:
tab_obs = 'OBSFLO'
dic_obs = 'DICFLO'
obs_flo = _get_obs(db_file, tab_obs, dic_obs, detail)
obs_flo = __insert_type_esp__(obs_flo, 'Flore')
if 'fish' == type_obs or 'fish' in type_obs:
tab_obs = 'OBSFSH'
dic_obs = 'DICFSH'
obs_fsh = _get_obs(db_file, tab_obs, dic_obs, detail)
obs_fsh = __insert_type_esp__(obs_fsh, 'Poisson')
if 'invert' == type_obs or 'invert' in type_obs:
tab_obs = 'OBSINV'
dic_obs = 'DICINV'
obs_inv = _get_obs(db_file, tab_obs, dic_obs, detail)
obs_inv = __insert_type_esp__(obs_inv, 'Invertébré')
if 'mamm' == type_obs or 'mamm' in type_obs:
tab_obs = 'OBSMAM'
dic_obs = 'DICMAM'
obs_mam = _get_obs(db_file, tab_obs, dic_obs, detail)
obs_mam = __insert_type_esp__(obs_mam, 'Mammifère')
if 'rept' == type_obs or 'rept' in type_obs:
tab_obs = 'OBSREP'
dic_obs = 'DICREP'
obs_rep = _get_obs(db_file, tab_obs, dic_obs, detail)
obs_rep = __insert_type_esp__(obs_rep, 'Reptile')
df_obs = pd.concat([obs_amp,obs_brd,obs_flo,obs_fsh,obs_inv,obs_mam,obs_rep])
df_obs = _del_na_col(df_obs)
df_rgl = __get_DicGenREGLE__()
df_aut = __get_auteur__(db_file)
if 'REGLE_COD' in df_obs.columns:
df_obs = pd.merge(df_obs,df_rgl,how='left', left_on='REGLE_COD', right_on='CODE')
df_obs.rename(columns={'DESCR':'NIV_PROTECT', 'CODE': 'CD_PROTECT'}, inplace=True)
if not detail:
df_obs.drop(columns=['REGLE_COD', 'CD_PROTECT'], inplace=True)
df_obs = pd.merge(df_obs,df_aut,how='left', left_on='AUTEUR', right_on='CODE')
df_obs.rename(columns={'DESCR':'NOM_AUT', 'CODE': 'CD_AUT'}, inplace=True)
if not detail:
df_obs.drop(columns=['AUTEUR', 'CD_AUT'], inplace=True)
return df_obs
########################################
### Fonctionnement de la zone humide ###
########################################
def __get_DicGenIN1__():
d = {'CODE':['1','2','3','4','5','6','7','8','9'], 'DESCR':[
'mer/océan', "cours d'eau",
'Canaux/fossés','Sources',
'nappes','précipitations',
"plans d'eau",'Ruissellement diffus',
'Eaux de crues',
]}
return pd.DataFrame(data=d)
def __get_DicGenIN2__():
d = {'CODE':['1','2','3','4'], 'DESCR':[
'permanent','saisonnier',
'temporaire','intermittent',
]}
return pd.DataFrame(data=d)
def __get_DicGenOUT__():
d = {'CODE':['1','2','3','4','5','6','7'], 'DESCR':[
'mer/océan', "cours d'eau",
'Canaux/fossés','Pompage, drainage',
'nappes','évaporation',
"plans d'eau",
]}
return pd.DataFrame(data=d)
def __get_DicGenOUT2__():
d = {'CODE':['1','2','3','4','5'], 'DESCR':[
'aucune','permanent',
'saisonnier','temporaire',
'intermittent',
]}
return pd.DataFrame(data=d)
def __get_DicGenFREQ__():
d = {'CODE':['1','2','3','4'], 'DESCR':[
'Jamais submergé','Toujours submergé',
'Exceptionnellement submergé',
'Régulièrement submergé',
]}
return pd.DataFrame(data=d)
def __get_DicGenETEND__():
d = {'CODE':['1','2'], 'DESCR':[
'Totalement submergé',
'Partiellement submergé',
]}
return pd.DataFrame(data=d)
def __get_DicGenCONNEX__():
d = {'CODE':['1','2','3','4','5','6'], 'DESCR':[
'traversée',
'entrée et sortie',
'entrée',
'sortie',
'passe à coté',
'aucune connexion',
], 'NOMFICH':[
'MWD1.JPG','MWD2.JPG','MWD3.JPG','MWD4.JPG','MWD5.JPG','MWD6.JPG']}
return pd.DataFrame(data=d)
def get_regHydroEntree(db_file, detail=False):
'''
:db_file: str. Nom de la base de données access
:detail: bool.
'''
df_in = mdb.read_table(db_file, 'SITEINFL')
df_dic1= __get_DicGenIN1__()
df_dic2= __get_DicGenIN2__()
df_in = pd.merge(df_in,df_dic1, how='left', left_on='INFLOW1', right_on='CODE')
df_in.rename(columns={'CODE':'CD_ENTREE_EAU', 'DESCR':'ENTREE_EAU'}, inplace=True)
df_in = pd.merge(df_in,df_dic2, how='left', left_on='INFLOW2', right_on='CODE')
df_in.rename(columns={'CODE':'CD_PERM', 'DESCR':'PERM_IN'}, inplace=True)
if not detail:
df_in.drop(columns=['INFLOW1','INFLOW2','CD_ENTREE_EAU','CD_PERM'], inplace=True)
return df_in
def get_regHydroSortie(db_file, detail=False):
'''
:db_file: str. Nom de la base de données access
:detail: bool.
'''
df_out = mdb.read_table(db_file, 'SITEOUTF')
df_dic1= __get_DicGenOUT__()
df_dic2= __get_DicGenOUT2__()
df_out = pd.merge(df_out,df_dic1, how='left', left_on='OUTFLOW', right_on='CODE')
df_out.rename(columns={'CODE':'CD_SORTIE_EAU', 'DESCR':'SORTIE_EAU'}, inplace=True)
if not isinstance(df_out.PERMANENCE,str):
df_out.PERMANENCE.fillna(0, inplace=True)
df_out.PERMANENCE = df_out.PERMANENCE.astype(int).astype(str)
df_out = pd.merge(df_out,df_dic2, how='left', left_on='PERMANENCE', right_on='CODE')
df_out.rename(columns={'PERMANENCE': 'CD_PERM1','CODE':'CD_PERM2', 'DESCR':'PERM_OUT'}, inplace=True)
if not detail:
df_out.drop(columns=['OUTFLOW','CD_SORTIE_EAU','CD_PERM1','CD_PERM2'], inplace=True)
return df_out
def get_regSubmersion(db_file, detail=False):
'''
:db_file: str. Nom de la base de données access
:detail: bool.
'''
df_inf = __get_SiteInfo__(db_file)
df = df_inf[['SITE_COD', 'FREQUENCE', 'ORIGINE', 'ETENDUE']]
# df.rename(columns={'ORIGINE':'SUB_ORIG'}, inplace=True)
df_dic1 = __get_DicGenFREQ__()
df_dic2 = __get_DicGenETEND__()
df = pd.merge(df,df_dic1, how='left', left_on='FREQUENCE', right_on='CODE')
df.rename(columns={'ORIGINE':'SUB_ORIG','FREQUENCE':'CD_FREQ1' ,'CODE':'CD_FREQ2', 'DESCR':'SUB_FREQ'}, inplace=True)
df = pd.merge(df,df_dic2, how='left', left_on='ETENDUE', right_on='CODE')
df.rename(columns={'ETENDUE':'CD_ETEND1' ,'CODE':'CD_ETEND2', 'DESCR':'SUB_ETEND'}, inplace=True)
if not detail:
df.drop(columns=['CD_FREQ1','CD_FREQ2','CD_ETEND1','CD_ETEND2'], inplace=True)
return df
def get_connex(db_file, detail=False):
df_inf = __get_SiteInfo__(db_file)
df_inf = df_inf[['SITE_COD', 'CONNEX']]
df_dic = __get_DicGenCONNEX__()
df = pd.merge(df_inf, df_dic, how='left', left_on='CONNEX', right_on='CODE')
df.rename(columns={'CODE':'CD_CONNEX', 'DESCR':'CONNEXION'}, inplace=True)
if not detail:
df.drop(columns=['CONNEX','CD_CONNEX','NOMFICH'], inplace=True)
return df
def get_fctmt_zh(db_file, detail=False):
df_in = get_regHydroEntree(db_file, detail)
if df_in.COORD_X.max() == 0 : df_in.drop(columns=['COORD_X'], inplace=True)
if df_in.COORD_Y.max() == 0 : df_in.drop(columns=['COORD_Y'], inplace=True)
df_out = get_regHydroSortie(db_file, detail)
if df_out.COORD_X.max() == 0 : df_out.drop(columns=['COORD_X'], inplace=True)
if df_out.COORD_Y.max() == 0 : df_out.drop(columns=['COORD_Y'], inplace=True)
df_sub = get_regSubmersion(db_file, detail)
df_con = get_connex(db_file, detail)
df_sub_con = pd.merge(df_sub,df_con, how='outer', on='SITE_COD')
lst_df = {
'entree_eau': df_in,
'sortie_eau': df_out,
'sub_connex': df_sub_con,
}
return lst_df
######################################
### Delimitation de la zone humide ###
######################################
def __get_DicGenLIM__():
d = {'CODE':['1','2','3','4','5','6','7'], 'DESCR': [
"hydrologie (balancement des eaux, crues, zones d'inondation, fluctuation de la nappe)",
'présence ou absence de sols hydromorphes',
"présence ou absence d'une végétation hygrophile",
'périodicité des inondations ou saturation du sol en eau',
'occupation des terres (limite entre les espaces naturels et les milieux anthropisés)',
'répartition et agencement spatial des habitats (types de milieux)',
'fonctionnement écologique (espace nécessaire à la biologie des espèces : connexions biologiques, relations entre écosystèmes)'
]}
return pd.DataFrame(data=d)
def __get_DicGenLIM_1__():
d = {'CODE':['1','2','3','4','5','6','7','8','9','10','11','12'], 'DESCR':[
'limites du bassin ou sous-bassin versant',
'limites des zones inondables',
"bassin d'alimentation souterrain",
"zone de recharge d'une nappe",
'occupation du sol',
'formations végétales, étages de végétation',
'limites paysagères',
'répartition et agencement spatial des habitats (types de milieux)',
"zone nécessaire à la vie d'une espèce",
'espace de transition entre des zones humides',
'zone humide altérée en partie ou totalement, restaurable',
'non déterminé',
]}
return pd.DataFrame(data=d)
def _get_espFct(db_file, detail=False):
'''
:db_file: str. Nom de la base de données access
:detail: bool.
'''
df = mdb.read_table(db_file, 'EFLIM')
if not isinstance(df.LIM1_COD,str):
df.LIM1_COD = df.LIM1_COD.astype(str)
df_dic = __get_DicGenLIM_1__()
df = pd.merge(df,df_dic, how='left', left_on='LIM1_COD', right_on='CODE')
df.rename(columns={'SIT_COD':'SITE_COD', 'DESCR':'DEF_ESPACE_FTC'}, inplace=True)
if not detail:
df.drop(columns=['LIM1_COD','CODE'], inplace=True)
return df
def _get_delim(db_file, detail=False):
'''
:db_file: str. Nom de la base de données access
:detail: bool.
'''
df = mdb.read_table(db_file, 'SITELIM')
if not isinstance(df.LIM_COD,str):
df.LIM_COD = df.LIM_COD.astype(str)
df_dic = __get_DicGenLIM__()
df = pd.merge(df,df_dic, how='left', left_on='LIM_COD', right_on='CODE')
df.rename(columns={'SIT_COD':'SITE_COD', 'DESCR':'CRIT_DELIM'}, inplace=True)
if not detail:
df.drop(columns=['LIM_COD','CODE'], inplace=True)
return df
def get_Delim_espaceFct(db_file, detail=False):
df_espFct = _get_espFct(db_file, detail)
df_delim = _get_delim(db_file, detail)
df = pd.merge(df_espFct,df_delim, how='left', on='SITE_COD')
return df
#####################################
### Description de la zone humide ###
#####################################
def __get_DicGenIMP__():
d = {'CODE':[
'0','10','11.0','12.0','13.0','14.0','15.0','16.0','17.0','20','21.0','22.0','23.0','24.0',
'30','31.0','32.0','33.0','34.0','35.0','36.0','37.0','38.0','40','41.0','42.0','43.0','44.0',
'45.0','46.0','47.4','48.0','50','51.0','52.0','53.0','54.0','55.0','61.0','62.0','63.0','64.0',
'70','71.0','72.0','73.0','74.0','75','75.0','76.0','77.0','78.0','79.0','80.0','81.0','82.0',
'83.0','84.0','85.0 ','86.0','90','91.0','91.1','91.2 ','91.3','91.4','91.5',
], 'DESCR':[
'AUCUN',
"IMPLANTATION, MODIFICATION OU FONCTIONNEMENT D'INFRASTRUCTURES ET AMENAGEMENTS LOURDS",
'habitats humain, zone urbanisée','zone industrielle ou commerciale',
'infrastructure linéaire, réseaux de communication',
'extraction de matériaux','dépôt de matériaux, décharge',
'équipement sportif et de loisirs','Infrastructure et équipement agricoles',
'POLLUTIONS ET NUISANCES',
'rejets substances polluantes dans les eaux','rejets substances polluantes dans les sols',
"rejets substances polluantes dans l'atmosphère",'nuisances liées à la surfréquentation, au piétinement',
'PRATIQUES LIÉES À LA GESTION DES EAUX',
'comblement, assèchement, drainage, poldérisation des zones humides',
"mise en eau, submersion, création de plan d'eau",'modification des fonds, des courants',
'création ou modification des berges et des digues, îles et îlots artificiels, remblais et déblais, fossés',
"entretien rivières, canaux, fossés, plan d'eau",'modification du fonctionnement hydraulique',
'action sur la végétation immergée, flottante ou amphibie, y compris faucardage et démottage',
'pêche professionnelle',
'PRATIQUES AGRICOLES ET PASTORALES',
'mise en culture, travaux du sol',
'débroussaillage, suppression haies et bosquets, remembrement et travaux connexes',
'jachère, abandon provisoire','traitement de fertilisation et pesticides','pâturage',
'suppression ou entretien de la végétation fauchage et fenaison',
'abandon de systèmes culturaux et pastoraux, apparition de friches',
'plantation de haies et de bosquets',
'PRATIQUES ET TRAVAUX FORESTIERS',
'coupes, abattages, arrachages et déboisements','taille, élagage',
'plantation, semis et travaux connexes',
'entretien liés à la sylviculture, nettoyage, épandage',
'autre aménagement forestier, accueil du public, création de pistesPRATIQUES LIEES AUX LOISIRS',
'Sport et loisir de plein air','Chasse','Pêche','Cueillette et ramassage',
"PRATIQUES DE GESTION OU D'EXPLOITATION DES ESPÈCES ET HABITATS",
'prélèvement sur la faune ou la flore',
'introduction, gestion ou limitation des populations',
"gestion des habitats pour l'accueil et l'information du public",
"autre (préciser dans l'encart réservé aux remarques)",
'PRATIQUES AQUACOLES',
'aménagements liés à la pisciculture ou à la conchyliculture',
'fertilisation, amendements','alimentation artificielle','rejets de déchets',
'vidanges',
'PROCESSUS NATURELS ABIOTIQUES',
'érosion','atterrissement, envasement, assèchement','submersion',
'mouvement de terrain','incendie','catastrophe naturelle',
'PROCESSUS BIOLOGIQUES ET ÉCOLOGIQUES',
'évolution écologique, appauvrissement, enrichissement',
'atterrissement','eutrophisation','acidification',"envahissement d'une espèce",
'fermeture du milieu',
]}
return pd.DataFrame(data=d)
def __get_DicGenPOS__():
d = {'CODE':['0','1','2','3'], 'DESCR':[
'inconnu','au niveau de la zone humide',
"au niveau de l'espace de fonctionnalité",
"au niveau de la zone humide et de l'espace de fonctionnalité",
]}
return pd.DataFrame(data=d)
def __get_DicACT__(db_file):
return mdb.read_table(db_file, 'DICACT')
def __get_DicGenSDA__():
d = {'CODE':['0','01','02','03','04','05','06','07','08','9','10','11','12','13'], 'DESCR':[
'None',
'grands estuaires',
'baies et estuaires moyens-plats',
'marais et lagunes côtiers',
'marais saumâtres aménagés',
"bordures de cours d'eau",
'plaines alluviales',
'zones humides de bas-fond en tête de bassin versant',
"régions d'étangs",
"petits plans d'eau et bordures de plans d'eau",
'marais et landes humides de plaines et plateaux',
'zones humides ponctuelles',
'marais aménagés dans un but agricole',
'zones humides artificielles',
]}
return pd.DataFrame(data=d)
def _get_SITEACT(db_file, detail=False):
df = mdb.read_table(db_file, 'SITEACT')
df = _del_na_col(df)
df_dic1 = __get_DicGenIMP__()
df_dic2 = __get_DicGenPOS__()
df_dic3 = __get_DicACT__(db_file)
if 'IMPACT_COD' in df.columns:
if not isinstance(df.IMPACT_COD,str):
df.IMPACT_COD = df.IMPACT_COD.astype(str)
df = pd.merge(df,df_dic1, how='left', left_on='IMPACT_COD', right_on='CODE')
df.rename(columns={'CODE':'CD_IMP', 'DESCR':'IMPACT'}, inplace=True)
if not detail:
df.drop(columns=['IMPACT_COD','CD_IMP'],inplace=True)
df = pd.merge(df,df_dic2, how='left', left_on='POSITION', right_on='CODE')
df.rename(columns={'CODE':'CD_LOC', 'DESCR':'LOCALISATION'}, inplace=True)
df = pd.merge(df,df_dic3, how='left', left_on='ACTIV_COD', right_on='CODE')
df.rename(columns={'CODE':'CD_ACTIV', 'DESCR':'ACTIVITE_HUM'}, inplace=True)
if not detail:
df.drop(
columns=['COORD_X','COORD_Y','POSITION','CD_LOC','ACTIV_COD','CD_ACTIV'],
inplace=True)
return df
def _get_SITEIMP(db_file, detail=False):
df = mdb.read_table(db_file, 'SITEIMP')
df = _del_na_col(df)
df_dic1 = __get_DicGenIMP__()
df_dic2 = __get_DicGenPOS__()
df_dic3 = __get_DicACT__(db_file)
if 'IMPACT_COD' in df.columns:
if not isinstance(df.IMPACT_COD,str):
df.IMPACT_COD = df.IMPACT_COD.astype(str)
df = pd.merge(df,df_dic1, how='left', left_on='IMPACT_COD', right_on='CODE')
df.rename(columns={'CODE':'CD_IMP', 'DESCR':'IMPACT'}, inplace=True)
if not detail:
df.drop(columns=['IMPACT_COD','CD_IMP'],inplace=True)
# df = pd.merge(df,df_dic2, how='left', left_on='POSITION', right_on='CODE')
# df.rename(columns={'CODE':'CD_LOC', 'DESCR':'LOCALISATION'}, inplace=True)
df = pd.merge(df,df_dic3, how='left', left_on='ACTIV_COD', right_on='CODE')
df.rename(columns={'CODE':'CD_ACTIV', 'DESCR':'ACTIVITE_HUM'}, inplace=True)
if not detail:
df.drop(
columns=[
# 'POSITION','CD_LOC',
'ACTIV_COD','CD_ACTIV'],
inplace=True)
return df
def get_usage_process(db_file, detail=False):
df_sitImp = _get_SITEIMP(db_file, detail)
df_sitAct = _get_SITEACT(db_file, detail)
if 'IMPACT' not in df_sitAct.columns:
df_sitAct.merge(df_sitImp, how='left', on=['SITE_COD', 'ACTIVITE_HUM'])
# Complexe..........
# df = (pd.concat([df_sitAct,df_sitImp])
# .sort_values(by=['SITE_COD','ACTIVITE_HUM','IMPACT'], na_position='last')
# .drop_duplicates(subset=['SITE_COD','ACTIVITE_HUM','IMPACT'], keep='first')
# .reset_index(drop=True))
# dup = df[df.duplicated(subset=['SITE_COD','ACTIVITE_HUM'])].index
# rmq = df[df.index.isin(dup)].REMARKS.dropna().index
# df.drop(dup).reset_index(drop=True)
return df_sitAct
def get_sdage(db_file, detail=False):
df_inf = __get_SiteInfo__(db_file)
df_inf = df_inf[['SITE_COD','SDAGE_COD']]
df_dic = __get_DicGenSDA__()
df = pd.merge(df_inf,df_dic, how='left', left_on='SDAGE_COD', right_on='CODE')
df.rename(columns={'CODE':'CD_SDAGE', 'DESCR':'TYPO_SDAGE'}, inplace=True)
if not detail:
df.drop(columns=['CD_SDAGE','SDAGE_COD'],inplace=True)
return df
def get_descrp_zh(db_file, detail=False):
df_usgP = get_usage_process(db_file, detail)
df_sdag = get_sdage(db_file, detail)
df_usgP = df_usgP.merge(df_sdag, how='outer', on=['SITE_COD'])
return df_usgP
#######################################
### Fonctions eclologiques, valeurs ###
### socio-économiques, interêt ###
### patrimonial de la zone humide ###
#######################################
def __get_DicGenTYPE__():
d = {'CODE':['1','2','3','4'], 'DESCR':[
'Fonctions hydrologiques',
'Fonctions biologiques',
'Valeurs socio-économiques',
'Intérêt patrimonial',
]}
return pd.DataFrame(data=d)
def __get_DicGenFVI__():
d = {
'CODE':['01','02','03','04','05','06','07','08','09','10',
'20','21','22','23','24','25','26','27','30','31',
'32','33','34','35','36','40','41','42','43','44',
'50','51','60','61','62','63','64','70'],
'TYPE': ['3','3','3','3','3','3','3','3','3','4','4','4',
'4','4','4','4','4','4','4','4','4','4','4','4','4',
'1','1','1','1','1','1','1','2','2','2','2','2','2'],
'DESCR':[
"réservoir pour l'alimentation en eau potable",
'production biologique (pâturage; fauche; sylviculture; aquaculture; pêche; chasse)',
'production de matière première (irrigation; granulats; énergie; tourbe; roseaux; etc.)',
'intérêt pour la valorisation pédagogique/éducation',
'intérêt paysager','intérêt pour les loisirs/valeurs récréatives',
'valeur scientifique','valeur culturelle',
'nuisances sur les conditions de vie des populations humaines résidentes (transmission parasitaire; moustiques; etc.)',
'HABITATS','FAUNISTIQUES','invertébrés (sauf insectes)','insectes','poissons',
'amphibiens','reptiles','oiseaux','mammifères','FLORISTIQUES','algues',
'champignons','lichens','bryophytes','ptéridophytes','phanérogames',
'FONCTIONS DE REGULATION HYDRAULIQUE',
'expansion naturelle des crues (contrôle des crues; écrêtement des crues; stockage des eaux de crues; prévention des inondations)',
'ralentissement du ruissellement',
"soutien naturel d'étiage (alimentation des nappes phréatiques; émergence des nappes phréatiques; recharge et protection des nappes phréatiques)",
"fonctions d'épuration (rétention de sédiments et de produits toxiques; recyclage et stockage de matière en suspension; régulation des cycles trophiques par exportation de matière organique; influence sur les cycles du carbone et de l'azote)",
'FONCTIONS DE PROTECTION DU MILIEU PHYSIQUE',"rôle naturel de protection contre l'érosion",
"FONCTION D'HABITAT POUR LES POPULATIONS ANIMALES OU VEGETALES",
"connexions biologiques (continuité avec d'autres milieux naturels), zone d'échanges, zone de passages, corridor écologique (faune, flore)",
'étapes migratoires, zones de stationnement, dortoirs',
"zone particulière d'alimentation pour la faune",'zone particulière liée à la reproduction',
"AUTRE INTERET FONCTIONNEL D'ORDRE ECOLOGIQUE (préciser dans l'encart réservé aux remarques)",
]}
return pd.DataFrame(data=d)
def _get_siteFVI(db_file, tab_obs):
return mdb.read_table(db_file,tab_obs)
def get_fct_zh(db_file, type_fct='all', detail=False):
'''
:db_file: str. Nom de la base de données access
:type_fct: str ou list. Code des observations à extraire de
la bdd access MEDWET : 'amphi', 'bird', 'flore', 'fish',
'invert', 'mamm', 'rept'. 'all' si tous.
:detail: bool.
'''
df_vib = pd.DataFrame()
df_vih = pd.DataFrame()
df_vis = pd.DataFrame()
df_vip = pd.DataFrame()
if type_fct == 'all' or 'all' in type_fct:
type_fct = ['hydro', 'bio', 'socio-eco', 'patri']
if 'bio' == type_fct or 'bio' in type_fct:
tab_obs = 'SITEFVIB'
df_vib = _get_siteFVI(db_file, tab_obs)
if 'hydro' == type_fct or 'hydro' in type_fct:
tab_obs = 'SITEFVIH'
df_vih = _get_siteFVI(db_file, tab_obs)
if 'socio-eco' == type_fct or 'socio-eco' in type_fct:
tab_obs = 'SITEFVIS'
df_vis = _get_siteFVI(db_file, tab_obs)
if 'patri' == type_fct or 'patri' in type_fct:
tab_obs = 'SITEFVIP'
df_vip = _get_siteFVI(db_file, tab_obs)
df = pd.concat([df_vib,df_vih,df_vis,df_vip])
df_dic1 = __get_DicGenFVI__()
df_dic2 = __get_DicGenTYPE__()
df = pd.merge(df,df_dic1, how='left', left_on='FVI_COD', right_on='CODE')
df.rename(columns={'CODE': 'CD_FVI','DESCR':'FONCTION'}, inplace=True)
df = pd.merge(df,df_dic2, how='left', left_on='TYPE', right_on='CODE')
df.rename(columns={'CODE': 'CD_TYPE','DESCR':'TYPE_FCT'}, inplace=True)
if not detail:
df.drop(columns=['FVI_COD','CD_FVI','CD_TYPE','TYPE'],inplace=True)
cols = df.columns
lst_memo = cols[cols.str.startswith('MEMO')]
df = _aggr_cols(df, lst_memo)
df.rename(columns={'aggreg': 'MEMO'}, inplace=True)
df.drop(columns=lst_memo, inplace=True)
return df
########################################
### CORINE BIOTOPE de la zone humide ###
########################################
def get_cb(db_file, detail=False):
df_cb = mdb.read_table(db_file, 'CATCHCBiot')
df_dic = mdb.read_table(db_file, 'DicGen-CBio')
df = pd.merge(df_cb,df_dic, how='left', left_on='CB_COD', right_on='CODE')
df.rename(columns={'DESCR':'DESC_CBIOTOPE'}, inplace=True)
if not detail:
df.drop(columns=['CODE'],inplace=True)
return df
#############################################
### Evaluation générale de la zone humide ###
#############################################
# get_eval_glob()
def get_eval_glob(db_file):
df_inf = __get_SiteInfo__(db_file)
df = df_inf[['SITE_COD','SITE_RAP1','SITE_RAP2','SITE_RAP3','SITE_RAP4']]
df.columns = ['SITE_COD', 'FCT_VALEURS_MAJEURS', 'INT_PATR_MAJEUR','BILAN_MENACES_FACTEURS', 'ORIENT_ACTIONS']
return df
#####################################
### Référence de la zone humide ###
#####################################
def __get_MWDREF__(db_file):
df = mdb.read_table(db_file, 'MWDREF')
df = _del_na_col(df)
return df
def get_biblio(db_file, detail=False):
df_ref = mdb.read_table(db_file, 'SITEREF')
if not isinstance(df_ref.REF_NO,str):
df_ref.REF_NO = df_ref.REF_NO.astype(str)
df_dic = __get_MWDREF__(db_file)
df = pd.merge(df_ref,df_dic, how='left', on='REF_NO')
if not detail:
df.drop(columns=['REF_NO'],inplace=True)
return df
#####################################
### Information de la zone humide ###
#####################################
def __get_CATCHINF__(db_file):
df = mdb.read_table(db_file, 'CATCHINF')
df = df[~df.CATCH_NAME.isna()]
df = df[['CATCH_COD', 'CATCH_NAME']]
return df
def get_SiteInfo(db_file, detail=False):
"""Récupération des informations génériques
des zones humides.
Paramètres :
------------
db_file : string. Chemin d'accès ver la bdd access MEDWET.
detail : bool. Affichage des relations entre les tables.
"""
# lst_col = [
# 'SITE_COD','DEPT', 'ORG','NUM','SITE_NAME','OTHER_NAME','COMPILER',
# 'EDITDATE','UPDDATE','AREA_WET','COORD_X','COORD_Y','SITE_RAP1','SITE_RAP2',
# 'SITE_RAP2','SITE_RAP3','SITE_RAP4','OTHER3_COD','OTHER4_COD','OTHER_INV',
# 'HISTORIQ','ZH_REMARKS','MEMO_ACTIV'] ###### NON FINI !! voir df_inf.columns
df_inf = __get_SiteInfo__(db_file)
# if 'CATCH_COD' in df_inf.columns:
# lst_col += ['CATCH_COD']
# df_inf = df_inf[lst_col]
df_inf.drop(columns=['CONNEX','FREQUENCE','ORIGINE','ETENDUE','SDAGE_COD',
'SITE_RAP1','SITE_RAP2','SITE_RAP3','SITE_RAP4'], inplace=True)
df_inf.rename(columns={
'OTHER1_COD':'CD_ZNIEFF_1G',
'OTHER2_COD':'CD_ZNIEFF_2G',
'OTHER3_COD':'CD_ZICO',
'OTHER4_COD':'CD_N2000',
}, inplace=True)
df_dic1 = __get_auteur__(db_file)
df_inf = df_inf.merge(df_dic1,how='left', left_on='COMPILER', right_on='CODE')
df_inf.rename(columns={'DESCR': 'AUTEUR'}, inplace=True)
if not detail:
df_inf.drop(columns=['COMPILER','CODE'], inplace=True)
if 'CATCH_COD' in df_inf.columns:
df_dic2 = __get_CATCHINF__(db_file)
df_inf = df_inf.merge(df_dic2,how='left', on='CATCH_COD')
if not detail:
df_inf.drop(columns=['CATCH_COD'],inplace=True)
return df_inf
if __name__ == "__main__":
# writer = pd.ExcelWriter('~/Documents/9_PROJETS/1_ZH/inventaire_zh_2021.xlsx', engine='xlsxwriter')
# workbook=writer.book
NAME_OUT = '/home/colas/Documents/13_CEN74/medwet2gn_ZH/inventaire_zh74_test.xlsx'
lst_fct = {
'Infos générales': get_SiteInfo,
'Corine Biotope' : get_cb,
'Délimitation de la zh': get_Delim_espaceFct,
'Description de la zh' : get_descrp_zh,
'Fonctmt de la zh': get_fctmt_zh,
'Fonctions de la zh' : get_fct_zh,
'Evaluation de la zh': get_eval_glob,
'Observations' : get_obs,
'Bibliographie': get_biblio
}
print('INIT récupération des données ...')
df = {}
for f,fonction in enumerate(lst_fct):
name = fonction
fct = lst_fct[fonction]
df1 = fct(db_file74)
# df2 = fct(db_file2)
# if f == 0:
# lst_stcd = df1[df1.SITE_COD.isin(df2.SITE_COD)].SITE_COD
# if fct == get_SiteInfo:
# df2.loc[df2.SITE_COD.isin(lst_stcd), 'OTHER_NAME'] = df1.loc[df1.SITE_COD.isin(lst_stcd),'SITE_NAME'].tolist()
if isinstance(df1, pd.DataFrame) :#and isinstance(df2, pd.DataFrame):
# df[f] = pd.concat([df1[~df1.SITE_COD.isin(lst_stcd)], df2])
df[f] = df1
df[f].name = name
elif isinstance(df1, dict) :#and isinstance(df2, dict):
df[f] = {}
df[f]['title'] = name
for d in df1:
# df[f][d] = pd.concat([df1[d][~df1[d].SITE_COD.isin(lst_stcd)], df2[d]])
df[f][d] = df1[d]
print('INIT écriture du fichier ...')
# Ecriture des données
with pd.ExcelWriter(NAME_OUT) as writer:
for d in df:
DF = df[d]
if isinstance(DF, pd.DataFrame):
DF.to_excel(writer,sheet_name=DF.name,startrow=1 , startcol=0, index=False, header=DF.columns)
ws = writer.book.active
writer.sheets[DF.name].cell(1,1,value=DF.name)
writer.save()
elif isinstance(DF, dict):
for i,d in enumerate(DF):
if d == 'title':
continue
if i == 1:
row = 1
col = 0
else:
col = DF[d].shape[1] + col + 3
DF[d].to_excel(writer,sheet_name=DF['title'],startrow=row , startcol=col, index=False)
ws = writer.book.active
writer.sheets[DF['title']].cell(column=col+1,row=row,value=d)
writer.save()
import sys
sys.exit('END SCRIPT ...')
# ws.cell(1,1,value=df_cb.name)
# writer.save()
# writer.write_cells(df_cb.name,sheet_name='infos_site',startrow=0,startcol=0)
# worksheet.write_string(df_inf.shape[0] + 4, 0, df2.name)
# df2.to_excel(writer,sheet_name='infos_site',startrow=df_inf.shape[0] + 5, startcol=0)
# df_inf = get_SiteInfo(db_file)
# df_inf.name = 'Infos générales'
# df_cb = get_cb(db_file)
# df_cb.name = 'Corine Biotope'
# df_evl = get_eval_glob(db_file)
# df_evl.name = 'Evaluation de la zh'
# df_dlm = get_Delim_espaceFct(db_file)
# df_dlm.name = 'Délimitation de la zh'
# df_dsc = get_descrp_zh(db_file)
# df_dsc.name = 'Description de la zh'
# df_ftm = get_fctmt_zh(db_file)
# df_ftm['title'] = 'Fonctmt de la zh'
# df_fct = get_fct_zh(db_file)
# df_fct.name = 'Fonctions de la zh'
# df_obs = get_obs(db_file)
# df_obs.name = 'Observations'
lst_df = [df_inf,df_cb,df_evl,df_dlm,df_dsc,df_ftm,df_fct,df_obs]
with pd.ExcelWriter(NAME_OUT) as writer:
for df in lst_df:
if isinstance(df, pd.DataFrame):
df.to_excel(writer,sheet_name=df.name,startrow=1 , startcol=0, index=False)
ws = writer.book.active
writer.sheets[df.name].cell(1,1,value=df.name)
writer.save()
elif isinstance(df, dict):
for i,d in enumerate(df):
if d == 'title':
continue
if i == 0:
row = 1
col = 0
else:
col = df[d].shape[1] + col + 3
df[d].to_excel(writer,sheet_name=df['title'],startrow=row , startcol=col, index=False)
ws = writer.book.active
writer.sheets[df['title']].cell(column=col+1,row=row,value=d)
writer.save()
import numpy as np
df1 = get_SiteInfo(db_file1)
df2 = get_SiteInfo(db_file2)
# df1 = get_fctmt_zh(db_file1)
# df2 = get_fctmt_zh(db_file2)
lst_stcd = df1[df1.SITE_COD.isin(df2.SITE_COD)].SITE_COD
# Get same columns
cols1 = df1.columns
cols2 = df2.columns
same_cols = cols1[cols1.isin(cols2)]
# tmp1 = df1.loc[df1.SITE_COD.isin(lst_stcd),same_cols].sort_values('SITE_COD').reset_index(drop=True)
# tmp2 = df2.loc[df2.SITE_COD.isin(lst_stcd),same_cols].sort_values('SITE_COD').reset_index(drop=True)
tmp1 = df1.loc[df1.SITE_COD.isin(lst_stcd),same_cols].sort_values('SITE_COD').set_index('SITE_COD',drop=True)
tmp2 = df2.loc[df2.SITE_COD.isin(lst_stcd),same_cols].sort_values('SITE_COD').set_index('SITE_COD',drop=True)
ne_stacked = (tmp1 != tmp2).stack()
changed = ne_stacked[ne_stacked]
changed.index.names = ['id', 'col']
difference_locations = np.where(tmp1 != tmp2)
changed_from = tmp1.values[difference_locations]
changed_to = tmp2.values[difference_locations]
pd.DataFrame({'from': changed_from, 'to': changed_to}, index=changed.index)
print(df1[df1.SITE_COD.isin(lst_stcd)].sort_values('SITE_COD').iloc[:,:15])
print(df2[df2.SITE_COD.isin(lst_stcd)].sort_values('SITE_COD').iloc[:,:15])
get_regHydroEntree(db_file)
get_regHydroSortie(db_file)
get_regSubmersion(db_file)
df = mdb.read_table(db_file, 'SITEINFO')
df = _del_na_col(df)
# Trouver un code parmis les tables dictionnaires
# paramètres
code = '26CCRV0012'
db_file = db_file1
# run
not_in = [
'SIG','DicGen','List','Switchboard','Items', # MEDWET2
'#Save_DicGen-FVI','~TMPCLP308581','#Save_SITEFVIS', # MEDWET1
'#Save_SITEFVIH','#Save_SITEFVIB','#Save_SITEFVIP', # MEDWET1
'$$$$1','$$$$2','#Save_SITEACT','$TEMP_ACTIV$', # MEDWET1
]
tmp = mdb.list_tables(db_file)
lst_t = [t for t in tmp if t.startswith('DicGen') and not t.endswith('TEMP')]
lst_t = [t for t in tmp]
for t in lst_t:
if t in not_in:
continue
df = mdb.read_table(db_file, t)
if not df.empty and 'SITE_COD' in df.columns:
boolean_findings = df.SITE_COD.str.contains(code)
total_occurence = boolean_findings.sum()
if(total_occurence > 0):
print(t)
elif not df.empty and not 'SITE_COD' in df.columns:
print('SITE_COD column is not "%s" '%t)
3_AZALEE/RHEZO-TUF/identification_tufiere.py
+31
View File
@@ -0,0 +1,31 @@
from pycen import zh,con_bdcen
import geopandas as gpd
zh = zh()
dzh = zh.v_zoneshumides()
col_date = dzh.columns[dzh.columns.str.contains('date')]
dzh[col_date] = dzh[col_date].astype(str)
site = gpd.read_postgis('SELECT * FROM sites.c_sites_zonages WHERE date_fin IS NULL',con_bdcen)
col_rmq = dzh.columns[dzh.columns.str.contains('rmq')]
c1 = dzh[dzh.code_cb.str.contains('54.1',na=False)]
c2 = c1[c1.intersects(site.unary_union)]
c3 = dzh[dzh.code_cb.str.contains('54.1') &
(
dzh.rmq_interet_patri.str.contains('tufi',case=False)
# dzh.rmq_site.str.contains('tufi',case=False) |
# dzh.rmq_fct_majeur.str.contains('tufi',case=False) |
# dzh.rmq_bilan_menace.str.contains('tufi',case=False) |
# dzh.rmq_orient_act.str.contains('tufi',case=False) |
# dzh.rmq_usage_process.str.contains('tufi',case=False) |
# dzh.rmq_activ_hum.str.contains('tufi',case=False)
)
]
to_file = '/home/colas/Documents/tmp/diagnos_tuffière.gpkg'
c1.to_file(to_file,layer='541 only',driver='GPKG')
c2.to_file(to_file,layer='541 ENS',driver='GPKG')
c3.to_file(to_file,layer='541 TUF',driver='GPKG')
3_AZALEE/azaleeTOgeonature.py
+1251
View File
File diff suppressed because it is too large Load Diff
3_AZALEE/create_tab_zh_ponctuel.py
+324
View File
@@ -0,0 +1,324 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-.
import geopandas as gpd
from pycen import con,zh
from os import path,listdir
from re import findall
v_zh = zh().v_zoneshumides()
PATH = '/media/cgeier/SRV/FICHIERS/OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/ETUDES/Zones humides'
lst_layer = [x for x in listdir(PATH) if (x.endswith('.shp') or x.endswith('.gpkg')) and x.startswith('ZHP')]
def upper_first_letter(s):
if s is None or gpd.pd.isna(s):
return s
else:
return s[0].upper() + s[1:]
def manual_replace(s, old, new, index):
if s is None or gpd.pd.isna(s):
return s
else:
if old == s[index]:
return s[:index] + new + s[index +1:]
else :
return s
def extract_info(df,value):
t1 = df[0].str.lower().str.contains(value,na=False)
t2 = df[1].str.lower().str.contains(value,na=False)
t3 = df[2].str.lower().str.contains(value,na=False)
t4 = df[3].str.lower().str.contains(value,na=False)
t5 = df[4].str.lower().str.contains(value,na=False)
df[5] = None
for i,t in enumerate([t1,t2,t3,t4,t5]):
if t.any():
df.loc[t,5] = df[t][i]
return df[5].str.strip()
def extract_descripti2(df):
df['statut'] = extract_info(df,'isit|possible ?|absence zh|non cartographier')
df['source'] = extract_info(df,'photointe|ortho|scan|information communale')
return df
if __name__ == "__main__":
gdf = gpd.GeoDataFrame()
for layer in lst_layer:
print(layer)
data = gpd.read_file(path.join(PATH,layer))
gdf = gpd.pd.concat([gdf,data])
gdf.DATEMODIF = gpd.pd.to_datetime(gdf.DATEMODIF,format='mixed')
gdf.columns = gdf.columns.str.lower()
gdf.sort_values(['datemodif'],inplace=True,ignore_index=True,ascending=False)
gdf.drop(columns=['X','Y'],inplace=True,errors='ignore')
gdf.rename_geometry('geom',inplace=True)
# Suppression des doublons de géometries - temps de calcul long
gdf.drop_duplicates(subset='geom',inplace=True,ignore_index=True,keep='first')
tmp = (gdf.descripti2
.str.replace('/','|')
.str.replace('06|19','06/19')
.str.replace('04|12|2019','04/12/2019')
.str.replace('vistier','visiter')
.str.replace('AbsenceZH','Absence ZH')
.str.replace('Zone humide non cartographiée et','non cartographier | ZH')
.str.replace('continuité zh de la gère, à visiter','continuité zh de la gère | à visiter')
.str.split('\\|| - ',expand=True)
.replace('',None))
res = extract_descripti2(tmp)
gdf[['statut','source']] = res[['statut','source']]
gdf['comment'] = (gdf.descripti2
.str.replace('vistier','visiter')
.str.replace('AbsenceZH','Absence ZH')
.str.replace('Zone humide non cartographiée et','non cartographier | ZH')
.str.replace('continuité zh de la gère, à visiter','continuité zh de la gère | à visiter')
.str.replace('à visiter','A visiter')
.replace([x for x in gdf.statut.unique() if x is not None],'',regex=True)
.replace([x for x in gdf.source.unique() if x is not None],'',regex=True)
.str.strip()
.replace('',None)
.apply(lambda x: manual_replace(x,'|','',0)).str.strip()
.apply(lambda x: manual_replace(x,'|','',0)).str.strip()
.replace('',None)
.apply(lambda x: manual_replace(x,'/','',0)).str.strip()
.apply(lambda x: manual_replace(x,'|','',len(x)-1) if x is not None else None)
.apply(lambda x: manual_replace(x,',','',len(x)-1) if x is not None else None)
.replace('',None)
.apply(lambda x: manual_replace(x,'/','',len(x)-1) if x is not None else None)
.str.strip()
.str.replace('| | ','|')
.replace('',None)
.apply(lambda x: manual_replace(x,'?','',len(x)-1) if x is not None else None)
.str.strip()
.apply(lambda x: manual_replace(x,'|','',len(x)-1) if x is not None else None)
.str.strip()
.str.replace('Extension zh amont - peupleraie','Peupleraie - extension zh amont')
.str.replace("Zone d'effondrement - source","Source - zone d'effondrement")
)
gdf['statut'] = (gdf['statut']
.apply(lambda x: upper_first_letter(x))
.str.replace(' ?','?')
.str.replace('?',' ?')
.replace('À visiter','A visiter'))
gdf['source'] = (gdf['source']
.apply(lambda x: upper_first_letter(x))
.str.replace('Interprétation','Photointerprétation'))
is_inventaire = (gdf.observateu
.str.contains('Le Pic Vert|LPO Isère|Lo Parvi|Nature Vivante|ONF',case=False,na=False))
isna_source = gdf['source'].isna()
is_cenisere = gdf.observateu.str.contains('cen isère',case=False,na=False)
gdf.loc[is_inventaire&isna_source,'source'] = 'Inventaire'
gdf.loc[~is_inventaire&isna_source&~is_cenisere].observateu
gdf['type'] = (gdf.comment
.str.replace('prairie','Prairie')
.str.replace('Prairie humide','Prairiehumide')
.str.split('\\|| ',n=1)
.str[0]
.str.replace('Prairiehumide','Prairie humide')
.str.strip()
.replace({
'Roselières':'Roselière',
'Ornières':'Ornière',
'Fontaine':'Source',
'Lac':'Mare/lac alpin',
'Culture':'Culture temporairement inondée',
'Grève':'Alluvion',
'ZH':'Tourbière',
})
.apply(lambda x: upper_first_letter(x))
)
gdf['comment'] = (gdf.comment
.str.replace('prairie','Prairie')
.replace([x for x in gdf.type.unique() if x is not None or x.contains('INVZH')],'',regex=True)
.str.replace('Prairie humide','Prairiehumide')
.str.split('\\|| ',n=1)
.str[1]
.str.replace('Prairiehumide','Prairie humide')
.str.strip()
.replace('',None)
.apply(lambda x: manual_replace(x,'-','',0)).str.strip()
)
et = gdf.comment.str.startswith('et ',na=False)
de = gdf.comment.str.startswith('de ',na=False)
gdf.loc[et,'comment'] = gdf.loc[et,'comment'].str.replace('et ','',n=1)
gdf.loc[de,'comment'] = gdf.loc[de,'comment'].str.replace('de ','',n=1)
invzh = gdf['type'].str.contains('INVZH',na=False)
gdf.loc[invzh,'comment'] = gdf.loc[invzh,'type']
gdf.loc[invzh,'type'] = None
gdf.comment = (gdf.comment
.str.strip()
.str.removeprefix('\|')
.str.strip()
.str.removeprefix('?')
.str.strip()
.str.removeprefix('-')
.str.removesuffix('-')
.str.strip()
.replace('',None)
.apply(lambda x: upper_first_letter(x)))
is_interpret = gdf['source'].str.contains('interprétation',case=False,na=False)
gdf.loc[is_interpret,'ponctuelle'] = 'Incertain'
gdf.loc[~is_interpret,'ponctuelle'] = 'Oui'
is_jlg = gdf.observateu=='CEN Isère (J.-L. Grossi)'
type_isna = gdf['type'].isna()
gdf.loc[is_jlg&type_isna,'type'] = 'Mare'
gdf.drop(gdf[gdf.observateu.isna()].index,inplace=True)
is_balm = gdf.observateu=='CEN Isère (C. Balmain)'
is_lafo = gdf.observateu=='CEN Isère (A.Lafon)'
is_nico = gdf.observateu=='CEN Isère (N. Biron)'
is_obs_inc = gdf.observateu=='Inconnu'
is_zh = gdf['type']=='Petite zh'
is_inconnu = gdf['type']=='Inconnu'
gdf.loc[is_balm&~is_zh&is_inconnu,'type'] = 'Étang'
gdf.loc[is_balm&is_zh&is_inconnu,'type'] = 'Suintement'
gdf.loc[is_lafo&is_inconnu,'type'] = 'Boisement'
gdf.loc[is_obs_inc&is_inconnu,'type'] = 'Étang'
gdf.loc[is_nico&is_inconnu,'type'] = 'Prairie humide'
is_visite = gdf['statut'].isin(['Possible ?','Groupement fontinal possible ?','Groupement fontinal, possible ?','Roselière et possible mare','Non cartographier'])
gdf.loc[is_visite,'statut'] = 'A visiter'
statut_isna = gdf['statut'].isna()
gdf.loc[is_interpret&statut_isna,'statut'] = 'A visiter'
is_zh2 = gdf['statut'].isna()&gdf['comment'].str.startswith('Zh',na=False)
gdf.loc[is_zh2,'statut'] = 'A décrire'
dict_type = {
**dict.fromkeys(['Barrage','Bassin','Lagunage'],'Artificiel'),
**dict.fromkeys(['Mégaphorbiaie','Prairie','Prairie humide','Roselière'],'Prairie humide et roselière'),
**dict.fromkeys(['Source', 'Suintement', 'Talus'],'Source et Suintement'),
**dict.fromkeys(['Ornière', 'Mare'],'Mare'),
**dict.fromkeys(['Boisement', 'Peupleraie'],'Boisement'),
}
gdf['classe'] = (gdf['type']
.replace(dict_type))
# Harmonisation des observateurs
dict_obs = {
'NB':'CEN Isère (N. Biron)',
'Nico':'CEN Isère (N. Biron)',
'Alix':'CEN Isère (A. Guedou)',
'N BIRON':'CEN Isère (N. Biron)',
'N. Biron (CEN Isère)':'CEN Isère (N. Biron)',
'Nicolas Biron':'CEN Isère (N. Biron)',
'M. Juton (CEN Isère)':'CEN Isère (M. Juton)',
'CEN Isère (J.-L. Grossi)':'CEN Isère (J. L. Grossi)',
'Lpo Isère (F. Frossard)':'LPO Isère (F. Frossard)',
'Lpo Isère (H. Coffre)':'LPO Isère (H. Coffre)',
'Lpo Isère (D. Loose)':'LPO Isère (D. Loose)',
'J. F. Noblet':'Le Pic Vert (J. F. Noblet)',
'R. Fonters':'LPO Isère (R. Fonters)',
'R. Marciau':'CEN Isère (R. Marciau)',
}
gdf.replace(dict_obs,inplace=True)
# Suppression des ZH taguer Absence
not_zh = gdf['statut']=='Absence ZH'
idx_notzh = gdf[not_zh].index
gdf.drop(idx_notzh,inplace=True)
# Intersection des zh_ponctuelles et zh_surfacique - temps de calcul long
v_zh_union = v_zh.unary_union
is_intersect = gdf.intersects(v_zh_union)
lst_drop = ['Tourbière','Culture','Boisement','Prairie humide','Roselière','Grève','Barrage','Lac']
drop_prospect_pontc = ~gdf['type'].isin(lst_drop)
gdf_zh = gdf[is_intersect&drop_prospect_pontc].copy()
gdf_ponct= gdf[~is_intersect].copy()
# A faire :
# - Intersecter avec la couche zh surfacique,
# dissocier les donner d'intersection et conserver
# dans une autre couche les zones ponctuelles correspondant à des Mares/Etangs/bassin/Lac
bdd_table = 'zh_ponctuelles'
gdf_ponct.to_postgis(bdd_table,con,'zones_humides',if_exists='replace',index=True if 'gid' not in gdf.columns else False,index_label='gid')
sql = """
ALTER TABLE zones_humides.{tab} ADD PRIMARY KEY (gid);
GRANT ALL ON TABLE zones_humides.{tab} TO grp_admin;
GRANT SELECT ON TABLE zones_humides.{tab} TO grp_consult;
""".format(tab=bdd_table)
with con.begin() as cnx:
cnx.execute(sql)
bdd_table = 'zh_ponctuelles_in_zh'
gdf_zh.to_postgis(bdd_table,con,'zones_humides',if_exists='replace',index=True if 'gid' not in gdf.columns else False,index_label='gid')
sql = """
ALTER TABLE zones_humides.{tab} ADD PRIMARY KEY (gid);
GRANT ALL ON TABLE zones_humides.{tab} TO grp_admin;
GRANT SELECT ON TABLE zones_humides.{tab} TO grp_consult;
""".format(tab=bdd_table)
with con.begin() as cnx:
cnx.execute(sql)
########### POTENTIELLE SURFACIQUE #############
sql = """
SELECT * FROM zones_humides.zh_ponctuelles;
"""
gdf = gpd.read_postgis(
sql = sql,
con = con)
is_interpret = gdf['source'].str.contains('interprétation',case=False,na=False)
del_notzh = ~(gdf['statut'] == 'Absence ZH')
avere = gdf[~is_interpret&del_notzh].copy()
is_visite = avere['statut'].isin(['Possible ?','Groupement fontinal, possible ?','Roselière et possible mare','Non cartographier'])
avere.loc[is_visite,'statut'] = 'A visiter'
dict_type = {
**dict.fromkeys(['Barrage','Bassin','Lagunage'],'Artificiel'),
**dict.fromkeys(['Mégaphorbiaie','Prairie','Prairie humide','Roselière'],'Prairie humide et roselière'),
**dict.fromkeys(['Source', 'Suintement', 'Talus'],'Source et Suintement'),
**dict.fromkeys(['Ornière', 'Mare'],'Mare'),
**dict.fromkeys(['Boisement', 'Peupleraie'],'Boisement'),
}
avere['classe'] = (gdf['type']
.replace(dict_type))
########### POTENTIELLE #############
ptent = gdf[is_interpret&del_notzh].copy()
# avere['a_visiter'] = None
avere.to_postgis('zh_ponctuelles_averee',con,'zones_humides',if_exists='replace',index_label='gid')
sql = """
ALTER TABLE zones_humides.zh_ponctuelles_averee ADD PRIMARY KEY (gid);
GRANT ALL ON TABLE zones_humides.zh_ponctuelles_averee TO grp_admin;
GRANT SELECT ON TABLE zones_humides.zh_ponctuelles_averee TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(sql)
test = """
SELECT * FROM zones_humides.zh_ponctuelles;
"""
data = gpd.read_postgis(
sql = test,
con = con)
3_AZALEE/create_view.py
+465
View File
@@ -0,0 +1,465 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-.
from sqlalchemy import create_engine, text
from sqlalchemy.engine import URL
from datetime import datetime as dt
import pandas as pd
import geopandas as gpd
# Parametres bdd
user = 'cen_admin'
pwd = '#CEN38@venir'
adr = '91.134.194.221'
port = '5432'
base = 'azalee'
url = URL.create('postgresql+psycopg2',
username=user,
password=pwd,
host=adr,
database=base,
)
con = create_engine(url)
v_zoneshumides = """
DROP VIEW IF EXISTS zones_humides.v_zoneshumides;
CREATE OR REPLACE VIEW zones_humides.v_zoneshumides
AS WITH temp1 as (
SELECT DISTINCT ON (s.id)
s.id,
max(s6.date) "date"
FROM sites.sites s
JOIN sites.type_milieu s3 ON s.id_type_milieu = s3.id
JOIN sites.r_sites_geom s6 ON s.id::text = s6.id_site::text
WHERE s.date_fin IS NULL
AND s3.nom_court::text = 'Milieux humides'::text
GROUP BY s.id
), hab as (
SELECT
s62.id_geom_site,
string_agg(s62.id_cb,';') code_cb,
string_agg(s621.lb_hab_fr,';') lib_cb
FROM zones_humides.r_site_habitat s62 JOIN ref_habitats.corine_biotope s621 ON s62.id_cb = s621.id
WHERE s62."valid"
GROUP BY 1
), usagprocess AS (
SELECT
s65.id_geom_site,
string_agg(s651.nom,';') activ_hum,
string_agg(s652.nom,';') impact,
string_agg(s653.nom,';') "position",
string_agg(s65.remarques,';') rmq_activ_hum
FROM zones_humides.r_site_usageprocess s65
JOIN zones_humides.param_activ_hum s651 ON s65.id_activ_hum = s651.id
JOIN zones_humides.param_impact s652 ON s65.id_impact = s652.id
JOIN zones_humides.param_position s653 ON s65.id_position = s653.id
WHERE s65."valid"
GROUP BY 1
), cnx as (
SELECT
s63.id_geom_site,
s631.nom connexion
FROM zones_humides.r_site_type_connect s63
JOIN zones_humides.param_type_connect s631 ON s63.id_param_connect = s631.id
WHERE s63."valid"
), tmp_auteur_site as (
SELECT
s.id,
string_agg(btrim(concat(s11.nom, ' ', s11.prenom,' (',COALESCE(s111.abbrev, s111.nom),')')),';') AS auteur_site
FROM sites.sites s
JOIN sites.type_milieu s3 ON s.id_type_milieu = s3.id
LEFT JOIN (sites.r_sites_auteur s1 JOIN (personnes.personne s11
JOIN personnes.organisme s111 ON s11.id_organisme = s111.id) ON s1.id_auteur = s11.id
) ON s.id::text = s1.id_site::text
WHERE s.date_fin IS NULL
AND s3.nom_court::text = 'Milieux humides'::text
GROUP BY s.id
), tmp_auteur_geom as (
SELECT
s6.id,
string_agg(btrim(concat(s611.nom, ' ', s611.prenom,' (',COALESCE(s6111.abbrev, s6111.nom),')')),';') AS auteur_geom
FROM sites.sites s
JOIN sites.type_milieu s3 ON s.id_type_milieu = s3.id
LEFT JOIN (sites.r_sites_geom s6
LEFT JOIN (sites.r_geomsites_auteur s61 JOIN (personnes.personne s611
JOIN personnes.organisme s6111 ON s611.id_organisme = s6111.id) ON s61.id_auteur = s611.id
) ON s6.id = s61.id_geom_site
) ON s.id = s6.id_site
WHERE s.date_fin IS NULL
AND s3.nom_court::text = 'Milieux humides'::text
GROUP BY s6.id
)
SELECT
s.id AS site_code,
s.nom,
s1.auteur_site,
s61.auteur_geom,
s6.geom,
s.date_deb AS date_site,
s6.date AS date_geom,
s3.nom_court AS type_milieu,
s4.nom AS type_site,
s5.nom AS typo_sdage,
s.remarques AS rmq_site,
s6.rmq_fct_majeur,
s6.rmq_interet_patri,
s6.rmq_bilan_menace,
s6.rmq_orient_act,
s6.rmq_usage_process,
s62.code_cb,
s62.lib_cb,
ct1.fct_bio,
ct1.fct_hydro,
ct1.int_patri,
ct1."val_socioEco",
ct2.crit_delim,
ct2.crit_def_esp,
s65.activ_hum,
s65.impact,
s65."position",
s65.rmq_activ_hum,
(string_to_array(ct3.entree_eau,'//'))[1] entree_eau_reg,
(string_to_array(ct3.entree_eau,'//'))[2] entree_eau_perm,
(string_to_array(ct3.entree_eau,'//'))[3] entree_eau_topo,
(string_to_array(ct3.sortie_eau,'//'))[1] sortie_eau_reg,
(string_to_array(ct3.sortie_eau,'//'))[2] sortie_eau_perm,
(string_to_array(ct3.sortie_eau,'//'))[3] sortie_eau_topo,
s63.connexion,
CASE WHEN s64."valid" THEN s64.id_origsub END origine_sub,
CASE WHEN s64."valid" THEN s641.nom END freq_sub,
CASE WHEN s64."valid" THEN s642.nom END etendu_sub
FROM sites.sites s
JOIN temp1 w USING (id)
LEFT JOIN tmp_auteur_site s1 ON s.id = s1.id
LEFT JOIN sites.type_milieu s3 ON s.id_type_milieu = s3.id
LEFT JOIN sites.type_site s4 ON s.id_type_site::text = s4.id::text
LEFT JOIN sites.typo_sdage s5 ON s.id_typo_sdage::text = s5.id::text
LEFT JOIN (sites.r_sites_geom s6
LEFT JOIN tmp_auteur_geom s61 ON s6.id = s61.id
LEFT JOIN hab s62 ON s6.id = s62.id_geom_site
LEFT JOIN cnx s63 ON s6.id = s63.id_geom_site
LEFT JOIN (zones_humides.r_site_sub s64
JOIN (zones_humides.param_sub s641 JOIN zones_humides.type_param_sub s6411 ON s6411.id = s641.id_type) ON s64.id_freqsub = s641.id
JOIN (zones_humides.param_sub s642 JOIN zones_humides.type_param_sub s6421 ON s6421.id = s642.id_type) ON s64.id_etendsub = s642.id
) ON s6.id = s64.id_geom_site
LEFT JOIN usagprocess s65 ON s6.id = s65.id_geom_site
LEFT JOIN crosstab(
'SELECT
s63.id_geom_site::bigint,
s632.nom_court type_param,
string_agg(
CASE WHEN s63.description IS NULL THEN s631.nom
WHEN s63.description = '''' THEN s631.nom
ELSE CONCAT(s631.nom,'' ('',s63.description,'')'') END,
'';'') fct
FROM zones_humides.r_site_fctecosociopatri s63
JOIN (zones_humides.param_fct_eco_socio_patri s631
JOIN zones_humides.type_param_fct s632 ON s631.id_type = s632.id
) ON s63.id_fct = s631.id
WHERE s63."valid"
GROUP BY 1,2
ORDER BY 1,2,3',
'SELECT DISTINCT s632.nom_court FROM zones_humides.r_site_fctecosociopatri s63
JOIN (zones_humides.param_fct_eco_socio_patri s631
JOIN zones_humides.type_param_fct s632 ON s631.id_type = s632.id
) ON s63.id_fct = s631.id
ORDER BY 1 DESC;'
) AS ct1 ("id_geom_site" bigint, "fct_bio" text, "fct_hydro" text, "int_patri" text, "val_socioEco" text) ON s6.id = ct1.id_geom_site
LEFT JOIN crosstab(
'SELECT
s64.id_geom_site::bigint,
s642.nom_court type_param,
string_agg(
CASE WHEN s64.description IS NULL THEN s641.nom
WHEN s64.description = '''' THEN s641.nom
ELSE CONCAT(s641.nom,'' ('',s64.description,'')'') END,
'';'') fct
FROM zones_humides.r_site_critdelim s64
JOIN (zones_humides.param_delim_fct s641
JOIN zones_humides.type_param_delim_fct s642 ON s641.id_type = s642.id
) ON s64.id_crit_delim = s641.id
WHERE s64."valid"
GROUP BY 1,2
ORDER BY 1,2,3'
) AS ct2 ("id_geom_site" bigint, "crit_delim" text, "crit_def_esp" text) ON s6.id = ct2.id_geom_site
LEFT JOIN crosstab(
'SELECT
s64.id_geom_site,
s64.in_out,
CONCAT(
--''reg_hydro : '',
string_agg(s641.nom,'';''),''//'',
--''permanence : '',
string_agg(s642.nom,'';''),''//'',
--''rmq_toponymie : '',
string_agg(s64.rmq_toponymie,'';'')
) hydro
FROM zones_humides.r_site_reghydro s64
LEFT JOIN zones_humides.param_reg_hydro s641 ON s64.id_reg_hydro = s641.id
LEFT JOIN zones_humides.param_permanence s642 ON s64.id_permanence = s642.id
WHERE s64."valid"
GROUP BY 1,2
ORDER BY 1,2',
'SELECT DISTINCT in_out FROM zones_humides.r_site_reghydro ORDER BY 1 DESC;'
) AS ct3 ("id_geom_site" bigint, "entree_eau" text, "sortie_eau" text) ON s6.id = ct3.id_geom_site
) ON s.id::text = s6.id_site::text
WHERE s.date_fin IS NULL
AND s3.nom_court::text = 'Milieux humides'::text
AND s6.date = w.date
--AND s63."valid" AND s64."valid"
ORDER BY s.id ASC NULLS FIRST;
"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zoneshumides TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zoneshumides TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(v_zoneshumides)
cnx.execute(grant)
v_pelouseseche = """
DROP VIEW IF EXISTS ps.v_pelouseseches;
CREATE OR REPLACE VIEW ps.v_pelouseseches
AS WITH temp1 as (
SELECT DISTINCT ON (s.id)
s.id,
max(s6.date) "date"
FROM sites.sites s
JOIN sites.type_milieu s3 ON s.id_type_milieu = s3.id
JOIN sites.r_sites_geom s6 ON s.id::text = s6.id_site::text
WHERE s.date_fin IS NULL
AND s3.nom_court::text = 'Pelouses sèches'::text
GROUP BY s.id
), hab as (
SELECT
id_geom_site,
date,
(string_to_array(hab1,'//'))[1] n_hab1,
(string_to_array(hab1,'//'))[2] cb_hab1,
(string_to_array(hab1,'//'))[3] lb_hab1,
(string_to_array(hab2,'//'))[1] n_hab2,
(string_to_array(hab2,'//'))[2] cb_hab2,
(string_to_array(hab2,'//'))[3] lb_hab2,
(string_to_array(hab3,'//'))[1] n_hab3,
(string_to_array(hab3,'//'))[2] cb_hab3,
(string_to_array(hab3,'//'))[3] lb_hab3,
(string_to_array(hab4,'//'))[1] n_hab4,
(string_to_array(hab4,'//'))[2] cb_hab4,
(string_to_array(hab4,'//'))[3] lb_hab4
FROM crosstab(
$$WITH tmp as (
SELECT
id_geom_site,
MAX("date") date
FROM ps.r_site_habitat
GROUP BY 1
)
SELECT
s61.id_geom_site,
s61.date,
s61.index,
CONCAT(s61.n_hab,'//',
s62.code_hab,'//',
s621.lb_hab_fr) hab
FROM ps.r_site_habitat s61
JOIN (ps.r_hab_cb s62 JOIN ref_habitats.corine_biotope s621 ON s62.code_hab = s621.id) ON s61.id = s62.id_sitehab
JOIN tmp USING (id_geom_site)
WHERE s61.date = tmp.date
ORDER BY 1,2,3;$$,
'SELECT DISTINCT index FROM ps.r_site_habitat'
) AS ct ("id_geom_site" bigint, "date" date, "hab1" text, "hab2" text, "hab3" text, "hab4" text)
), tmp_auteur_site as (
SELECT
s.id,
string_agg(btrim(concat(s11.nom, ' ', s11.prenom,' (',COALESCE(s111.abbrev, s111.nom),')')),';') AS auteur_site
FROM sites.sites s
JOIN sites.type_milieu s3 ON s.id_type_milieu = s3.id
LEFT JOIN (sites.r_sites_auteur s1 JOIN (personnes.personne s11
JOIN personnes.organisme s111 ON s11.id_organisme = s111.id) ON s1.id_auteur = s11.id
) ON s.id::text = s1.id_site::text
WHERE s.date_fin IS NULL
AND s3.nom_court::text = 'Pelouses sèches'::text
GROUP BY s.id
), tmp_auteur_geom as (
SELECT
s6.id,
string_agg(btrim(concat(s611.nom, ' ', s611.prenom,' (',COALESCE(s6111.abbrev, s6111.nom),')')),';') AS auteur_geom
FROM sites.sites s
JOIN sites.type_milieu s3 ON s.id_type_milieu = s3.id
LEFT JOIN (sites.r_sites_geom s6
LEFT JOIN (sites.r_geomsites_auteur s61 JOIN (personnes.personne s611
JOIN personnes.organisme s6111 ON s611.id_organisme = s6111.id) ON s61.id_auteur = s611.id
) ON s6.id = s61.id_geom_site
) ON s.id = s6.id_site
WHERE s.date_fin IS NULL
AND s3.nom_court::text = 'Pelouses sèches'::text
GROUP BY s6.id
)
SELECT DISTINCT ON (s.id)
s.id AS site_code,
s.nom,
s1.auteur_site,
s61.auteur_geom,
s6.geom,
s.date_deb AS date_site,
s6.date AS date_geom,
s3.nom_court AS type_milieu,
s4.nom AS type_site,
s.remarques AS rmq_site,
s6.rmq_fct_majeur,
s6.rmq_interet_patri,
s6.rmq_bilan_menace,
s6.rmq_orient_act,
s6.rmq_usage_process,
ct1.statut,
ct1.pratique,
ct1.recouvrement,
ct1.embrouss,
ct1.taux tx_embrouss,
hab.n_hab1,
hab.cb_hab1,
hab.lb_hab1,
hab.n_hab2,
hab.cb_hab2,
hab.lb_hab2,
hab.n_hab3,
hab.cb_hab3,
hab.lb_hab3,
hab.n_hab4,
hab.cb_hab4,
hab.lb_hab4
FROM sites.sites s
JOIN temp1 w USING (id)
LEFT JOIN tmp_auteur_site s1 ON s.id = s1.id
LEFT JOIN sites.type_milieu s3 ON s.id_type_milieu = s3.id
LEFT JOIN sites.type_site s4 ON s.id_type_site::text = s4.id::text
LEFT JOIN (sites.r_sites_geom s6
LEFT JOIN tmp_auteur_geom s61 ON s6.id = s61.id
LEFT JOIN hab ON s6.id = hab.id_geom_site
LEFT JOIN crosstab(
'SELECT
s63.id_geom_site::bigint,
s63.taux,
s632.nom,
s631.description
FROM ps.r_site_param s63
JOIN (ps.param s631
JOIN ps.type_param s632 ON s631.id_type = s632.id
) ON s63.id_param = s631.id
--GROUP BY 1,2
ORDER BY 1,2,3',
'SELECT nom FROM ps.type_param'
) AS ct1 ("id_geom_site" bigint, "taux" text, "recouvrement" text, "embrouss" text, "pratique" text, "statut" text) ON s6.id = ct1.id_geom_site
) ON s6.id_site = s.id
WHERE s.date_fin IS NULL AND s3.nom_court::text = 'Pelouses sèches'::text
ORDER BY s.id, s6.date DESC NULLS LAST
;"""
grant = """
GRANT ALL ON TABLE ps.v_pelouseseches TO grp_admin;
GRANT SELECT ON TABLE ps.v_pelouseseches TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(v_pelouseseche)
cnx.execute(grant)
###########################
######## TEST CODE ########
###########################
select_import = """
WITH temp1 as (
SELECT DISTINCT ON (s.id)
s.id,
max(s6.date) "date"
FROM sites.sites s
JOIN sites.type_milieu s3 ON s.id_type_milieu = s3.id
JOIN sites.r_sites_geom s6 ON s.id::text = s6.id_site::text
WHERE s.date_fin IS NULL
AND s3.nom_court::text = 'Pelouses sèches'::text
GROUP BY s.id
), tmp_auteur_site as (
SELECT
s.id,
string_agg(btrim(concat(s11.nom, ' ', s11.prenom,' (',COALESCE(s111.abbrev, s111.nom),')')),';') AS auteur_site
FROM sites.sites s
JOIN sites.type_milieu s3 ON s.id_type_milieu = s3.id
LEFT JOIN (sites.r_sites_auteur s1 JOIN (personnes.personne s11
JOIN personnes.organisme s111 ON s11.id_organisme = s111.id) ON s1.id_auteur = s11.id
) ON s.id::text = s1.id_site::text
WHERE s.date_fin IS NULL
AND s3.nom_court::text = 'Pelouses sèches'::text
GROUP BY s.id
), tmp_auteur_geom as (
SELECT
s6.id,
string_agg(btrim(concat(s611.nom, ' ', s611.prenom,' (',COALESCE(s6111.abbrev, s6111.nom),')')),';') AS auteur_geom
FROM sites.sites s
JOIN sites.type_milieu s3 ON s.id_type_milieu = s3.id
LEFT JOIN (sites.r_sites_geom s6
LEFT JOIN (sites.r_geomsites_auteur s61 JOIN (personnes.personne s611
JOIN personnes.organisme s6111 ON s611.id_organisme = s6111.id) ON s61.id_auteur = s611.id
) ON s6.id = s61.id_geom_site
) ON s.id = s6.id_site
WHERE s.date_fin IS NULL
AND s3.nom_court::text = 'Pelouses sèches'::text
GROUP BY s6.id
)
SELECT DISTINCT ON (s.id)
s.id AS site_code,
s.nom,
s1.auteur_site,
s61.auteur_geom,
s6.geom,
s.date_deb AS date_site,
s6.date AS date_geom,
s3.nom_court AS type_milieu,
s4.nom AS type_site,
s.remarques AS rmq_site,
s6.rmq_fct_majeur,
s6.rmq_interet_patri,
s6.rmq_bilan_menace,
s6.rmq_orient_act,
s6.rmq_usage_process
FROM sites.sites s
JOIN temp1 w USING (id)
LEFT JOIN tmp_auteur_site s1 ON s.id = s1.id
LEFT JOIN sites.type_milieu s3 ON s.id_type_milieu = s3.id
LEFT JOIN sites.type_site s4 ON s.id_type_site::text = s4.id::text
LEFT JOIN (sites.r_sites_geom s6
LEFT JOIN tmp_auteur_geom s61 ON s6.id = s61.id
LEFT JOIN crosstab(
'SELECT
s63.id_geom_site::bigint,
s63.taux tx_embrouss,
s632.nom type_param,
s631.description param
FROM ps.r_site_param s63
JOIN (ps.param s631
JOIN ps.type_param s632 ON s631.id_type = s632.id
) ON s63.id_param = s631.id
--GROUP BY 1,2
ORDER BY 1,2,3',
'SELECT nom FROM ps.type_param'
) AS ct1 ("id_geom_site" bigint, "taux" text, "statut" text, "embrouss" text, "pratique" text, "recouvremt" text) ON s6.id = ct1.id_geom_site
) ON s6.id_site = s.id
WHERE s.date_fin IS NULL AND s3.nom_court::text = 'Pelouses sèches'::text
ORDER BY s.id, s6.date DESC NULLS LAST
;
"""
select_import = """
SELECT * FROM zones_humides.v_zoneshumides
;
"""
data = gpd.read_postgis(
sql = text(select_import),
con = con,
geom_col='geom')
3_AZALEE/create_view_pers.py
+34
View File
@@ -0,0 +1,34 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-.
from sqlalchemy import create_engine, text
from sqlalchemy.engine import URL
from datetime import datetime as dt
import pandas as pd
import geopandas as gpd
# Parametres bdd
from pycen import con
v_perso = """
DROP VIEW IF EXISTS personnes.v_personne CASCADE;
CREATE OR REPLACE VIEW personnes.v_personne
AS
SELECT
a.id,
CASE WHEN btrim(lower(concat(a.nom, ' ', a.prenom))) = lower(COALESCE(b.abbrev, b.nom))
then btrim(concat(a.nom, ' ', a.prenom))
else string_agg(btrim(concat(a.nom, ' ', a.prenom,' (',COALESCE(b.abbrev, b.nom),')')),';')
end AS auteur
FROM personnes.personne a
JOIN personnes.organisme b ON a.id_organisme = b.id
GROUP BY a.id,b.abbrev, b.nom;
"""
grant = """
GRANT ALL ON TABLE personnes.v_personne TO grp_admin;
GRANT SELECT ON TABLE personnes.v_personne TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(text(v_perso))
cnx.execute(grant)
3_AZALEE/create_view_ps.py
+499
View File
@@ -0,0 +1,499 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-.
from sqlalchemy import create_engine, text
from sqlalchemy.engine import URL
from datetime import datetime as dt
import pandas as pd
import geopandas as gpd
from pycen import con
drop_v_ps = '''
DROP VIEW IF EXISTS ps.v_pelouseseches_all CASCADE;
DROP VIEW IF EXISTS ps.v_termophile CASCADE;
'''
with con.begin() as cnx:
cnx.execute(drop_v_ps)
v_ps_habcb = """
DROP VIEW IF EXISTS ps.v_ps_habcb;
CREATE OR REPLACE VIEW ps.v_ps_habcb
AS
SELECT
b.id_sitehab,
string_agg(
CASE
WHEN b.suspect is not null AND b.sep is not null THEN CONCAT(b.code_hab,' (',b.suspect,')',b.sep)
WHEN b.suspect is not null AND b.sep is null THEN CONCAT(b.code_hab,' (',b.suspect,')')
WHEN b.suspect is null AND b.sep is not null THEN CONCAT(b.code_hab,b.sep)
WHEN b.suspect is null AND b.sep is null THEN b.code_hab
ELSE b.code_hab
END,'' ORDER BY b.ordre) code_hab,
string_agg(b1.lb_hab_fr,';') lb_hab_fr
FROM ps.r_hab_cb b JOIN ref_habitats.corine_biotope b1 ON b.code_hab = b1.id
GROUP BY 1
ORDER BY 1
;
"""
grant = """
GRANT ALL ON TABLE ps.v_ps_habcb TO grp_admin;
GRANT SELECT ON TABLE ps.v_ps_habcb TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(text(v_ps_habcb))
cnx.execute(grant)
v_ps_sitehab_cross = """
DROP VIEW IF EXISTS ps.v_ps_sitehab_cross;
CREATE OR REPLACE VIEW ps.v_ps_sitehab_cross
AS
SELECT
id_site,
id_geom_site,
auteur,
date,
date_max,
(string_to_array(hab1,'//'))[1] n_hab1,
(string_to_array(hab1,'//'))[2] cb_hab1,
(string_to_array(hab1,'//'))[3] lb_hab1,
(string_to_array(hab2,'//'))[1] n_hab2,
(string_to_array(hab2,'//'))[2] cb_hab2,
(string_to_array(hab2,'//'))[3] lb_hab2,
(string_to_array(hab3,'//'))[1] n_hab3,
(string_to_array(hab3,'//'))[2] cb_hab3,
(string_to_array(hab3,'//'))[3] lb_hab3,
(string_to_array(hab4,'//'))[1] n_hab4,
(string_to_array(hab4,'//'))[2] cb_hab4,
(string_to_array(hab4,'//'))[3] lb_hab4
FROM crosstab(
$$
WITH auteur AS (
SELECT DISTINCT ON (id_sitehab)
c.id_sitehab,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM ps.r_sitehab_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_sitehab
)
SELECT
a.id_site,
a.id_geom_site,
c.auteur,
a.date,
(SELECT MAX(date) FROM ps.r_site_habitat WHERE id_geom_site = a.id_geom_site ) date_max,
a.index,
CONCAT(a.n_hab,'//',
b.code_hab,'//',
b.lb_hab_fr
) hab
FROM ps.r_site_habitat a
LEFT JOIN ps.v_ps_habcb b ON a.id = b.id_sitehab
LEFT JOIN auteur c ON c.id_sitehab = a.id
WHERE
a.date = (SELECT MAX(date) FROM ps.r_site_habitat WHERE id_geom_site = a.id_geom_site)
ORDER BY a.id_site,a.id_geom_site,a.index
;$$,
'SELECT DISTINCT index FROM ps.r_site_habitat ORDER BY index LIMIT 4 ;'
) AS ct (
"id_site" varchar(10),
"id_geom_site" bigint,
"auteur" text,
"date" date,
"date_max" date,
"hab1" text,
"hab2" text,
"hab3" text,
"hab4" text)
ORDER BY 1,2
"""
grant = """
GRANT ALL ON TABLE ps.v_ps_sitehab_cross TO grp_admin;
GRANT SELECT ON TABLE ps.v_ps_sitehab_cross TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(text(v_ps_sitehab_cross))
cnx.execute(grant)
# PB a.id_geom_site in (52716)
v_ps_params_cross = """
DROP VIEW IF EXISTS ps.v_ps_params_cross;
CREATE OR REPLACE VIEW ps.v_ps_params_cross
AS
SELECT --ct.*
ct.id_site,
ct.id_geom_site,
ct.auteur,
ct.date,
ct.date_max,
ct.statut,
ct.pratique,
ct.recouvrement,
ct.embrouss,
ct.taux tx_embrouss
FROM crosstab(
$$
WITH auteur AS (
SELECT DISTINCT ON (id_siteparam)
c.id_siteparam,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM ps.r_siteparam_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_siteparam
)
SELECT DISTINCT ON (a.id_site,a.id_geom_site, a2.nom,a.date)
a.id_site,
a.id_geom_site::bigint,
a.date,
(SELECT MAX(date) FROM ps.r_site_param WHERE id_geom_site = a.id_geom_site ) date_max,
STRING_AGG(distinct c.auteur,', '),
a.taux,
a2.nom,
STRING_AGG(a1.description,', ')
FROM ps.r_site_param a
JOIN (ps.param a1
JOIN ps.type_param a2 ON a1.id_type = a2.id
) ON a.id_param = a1.id
JOIN auteur c ON c.id_siteparam = a.id
--WHERE
-- a.id_geom_site in (52716)
GROUP BY 1,2,7,3,6
ORDER BY 1,2,7,3 DESC
$$,
$$SELECT nom FROM ps.type_param ORDER BY nom;$$
) AS ct ("id_site" varchar(10),"id_geom_site" bigint, "date" date, "date_max" date, "auteur" text, "taux" text, "embrouss" text, "pratique" text, "recouvrement" text, "statut" text)
--GROUP BY 1,2,4,5,6,7,8
ORDER BY 1,2
"""
grant = """
GRANT ALL ON TABLE ps.v_ps_params_cross TO grp_admin;
GRANT SELECT ON TABLE ps.v_ps_params_cross TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(text(v_ps_params_cross))
cnx.execute(grant)
v_pelouseseches_all = drop_v_ps + """
CREATE OR REPLACE VIEW ps.v_pelouseseches_all
AS WITH tmp_auteur_site as (
SELECT
s.id,
string_agg(s11.auteur,';' ORDER BY s11.auteur) AS auteur_site
FROM sites.sites s
JOIN sites.type_milieu s3 ON s.id_type_milieu = s3.id
LEFT JOIN (sites.r_sites_auteur s1
JOIN personnes.v_personne s11 ON s1.id_auteur = s11.id
) ON s.id::text = s1.id_site::text
WHERE s.date_fin IS NULL
AND s3.nom_court::text = 'Pelouses sèches'::text
GROUP BY s.id
ORDER BY s.id
), tmp_auteur_geom as (
SELECT
s6.id,
string_agg(s611.auteur,';' ORDER BY s611.auteur) AS auteur_geom
FROM sites.sites s
JOIN sites.type_milieu s3 ON s.id_type_milieu = s3.id
LEFT JOIN (sites.r_sites_geom s6
LEFT JOIN (sites.r_geomsites_auteur s61
JOIN personnes.v_personne s611 ON s61.id_auteur = s611.id
) ON s6.id = s61.id_geom_site
) ON s.id = s6.id_site
WHERE s.date_fin IS NULL
AND s3.nom_court::text = 'Pelouses sèches'::text
GROUP BY s6.id
ORDER BY s6.id
)
SELECT DISTINCT ON (s.id)
s.id AS site_code,
s6.geom,
s.nom,
s5.nom district_nat,
s6.libelle source,
s6.id_origine,
s1.auteur_site,
s6.auteur_geom,
(
SELECT DISTINCT ON ("date") auteur
FROM (values (param.date_max,param.auteur),(hab.date_max,hab.auteur)) AS value("date",auteur)
WHERE "date" = (SELECT MAX("date") FROM (values (param.date_max),(hab.date_max)) AS value("date"))
) as auteur_attrs,
s.date_deb AS date_site,
s6."date" AS date_geom,
(
SELECT MAX("date") FROM (values (param.date_max),(hab.date_max)) AS value("date")
) as date_attrs,
s3.nom_court AS type_milieu,
s4.nom AS type_site,
s.remarques AS rmq_site,
s6.rmq_fct_majeur,
s6.rmq_interet_patri,
s6.rmq_bilan_menace,
s6.rmq_orient_act,
s6.rmq_usage_process,
param.statut,
param.pratique,
param.recouvrement,
param.embrouss,
param.tx_embrouss,
hab.n_hab1,
hab.cb_hab1,
hab.lb_hab1,
hab.n_hab2,
hab.cb_hab2,
hab.lb_hab2,
hab.n_hab3,
hab.cb_hab3,
hab.lb_hab3,
hab.n_hab4,
hab.cb_hab4,
hab.lb_hab4
FROM sites.sites s
JOIN tmp_auteur_site s1 ON s.id = s1.id
JOIN sites.type_milieu s3 ON s.id_type_milieu = s3.id
LEFT JOIN sites.type_site s4 ON s.id_type_site::text = s4.id::text
JOIN ref_territoire.districts_naturels s5 ON s.org = s5.abrev
JOIN ( SELECT DISTINCT ON (id_site) *
FROM sites.r_sites_geom sg
JOIN tmp_auteur_geom s61 ON sg.id = s61.id
LEFT JOIN sites.lots s62 USING (id_lot)
ORDER BY id_site, "date" DESC, date_insert DESC
) s6 ON s6.id_site = s.id
LEFT JOIN ps.v_ps_sitehab_cross hab ON s.id = hab.id_site
LEFT JOIN ps.v_ps_params_cross param ON s.id = param.id_site
----LEFT JOIN (sites.r_sites_geom s6
---- LEFT JOIN tmp_auteur_geom s61 ON s6.id = s61.id
---- LEFT JOIN ps.v_ps_sitehab_cross hab ON s6.id = hab.id_geom_site
---- LEFT JOIN ps.v_ps_params_cross param ON s6.id = param.id_geom_site
----) ON s6.id_site = s.id
--LEFT JOIN ps.r_infeq_1200m s7 ON s.id = s7.id_site
WHERE s.date_fin IS NULL
AND s3.nom_court::text = 'Pelouses sèches'::text
--AND s3.id = 2
--AND s6.date = (SELECT MAX(date) FROM sites.r_sites_geom WHERE id_site = s.id)
--AND s6.date_insert = (
-- SELECT MAX(date_insert) FROM sites.r_sites_geom WHERE id_site = s.id AND date = (
-- SELECT MAX(date) FROM sites.r_sites_geom WHERE id_site = s.id))
--AND s7.infeq_1200 IS True
ORDER BY s.id, s6."date" DESC NULLS LAST
;"""
grant = """
GRANT ALL ON TABLE ps.v_pelouseseches_all TO grp_admin;
GRANT SELECT ON TABLE ps.v_pelouseseches_all TO grp_consult;
"""
comment = """
COMMENT ON VIEW ps.v_pelouseseches_all IS 'Vue des sites à pelouses sèches et milieux thermophiles';
COMMENT ON COLUMN ps.v_pelouseseches_all.site_code IS 'Identifiant de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_all.nom IS 'Nom de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_all.district_nat IS 'District naturel sur lequel est positionner la majeur partie de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_all.source IS 'Nom de la couche source de la pelouse sèche au sein du CEN Isère.';
COMMENT ON COLUMN ps.v_pelouseseches_all.id_origine IS 'Identifiant d''origine de la pelouse sèche au sein de la couche source.';
COMMENT ON COLUMN ps.v_pelouseseches_all.auteur_site IS 'Auteur ayant caractérisé la pelouse sèche pour la première fois.';
COMMENT ON COLUMN ps.v_pelouseseches_all.auteur_geom IS 'Auteur ayant définis la géometrie actuelle de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_all.auteur_attrs IS 'Auteur le plus récent ayant défini les attributs de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_all.date_site IS 'Date de la première description de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_all.date_geom IS 'Date de la géométrie.';
COMMENT ON COLUMN ps.v_pelouseseches_all.date_attrs IS 'Date des attributs les plus récents.';
COMMENT ON COLUMN ps.v_pelouseseches_all.type_milieu IS 'Caractérisation du milieu.';
COMMENT ON COLUMN ps.v_pelouseseches_all.type_site IS '';
COMMENT ON COLUMN ps.v_pelouseseches_all.rmq_site IS 'Remarques générale concernant la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_all.rmq_fct_majeur IS 'Remarques sur les fonctions majeurs de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_all.rmq_interet_patri IS 'Remarques sur les intérêts patrimoniaux de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_all.rmq_bilan_menace IS 'Remarques globales sur les menaces qui concernent la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_all.rmq_orient_act IS 'Remarques sur les orientations et les actes de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_all.rmq_usage_process IS 'Remarques concernant les usages et les processus naturels de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_all.statut IS 'Statut (Communautaire, Prioritaire ou non défini) de la pelouse sèche définit suivant la Directive Habitat.';
COMMENT ON COLUMN ps.v_pelouseseches_all.pratique IS 'Pratique agricole réalisée sur la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_all.recouvrement IS 'Niveau de recouvrement du sol herbacé par rapport au sol nu.';
COMMENT ON COLUMN ps.v_pelouseseches_all.embrouss IS 'Niveau d''embroussaillement de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_all.tx_embrouss IS 'Taux d''embroussaillement de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_all.n_hab1 IS 'Pourcentage de présence de l''habitat 1 sur 4 de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_all.cb_hab1 IS 'Identifiants CORINE Biotopes constituant l''habitat 1 sur 4 de la pelouse sèche.\n- & : Entremêlement d''habitat\n- / : Habitat en transition dans le sens de lecture\n- (..) : Habitat suspecté.';
COMMENT ON COLUMN ps.v_pelouseseches_all.lb_hab1 IS 'Libellé(s) CORINE Biotopes constituant(s) l''habitat 1 sur 4 de la pelouse sèche. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN ps.v_pelouseseches_all.n_hab2 IS 'Pourcentage de présence de l''habitat 2 sur 4 de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_all.cb_hab2 IS 'Identifiants CORINE Biotopes constituant l''habitat 2 sur 4 de la pelouse sèche.\n- & : Entremêlement d''habitat\n- / : Habitat en transition dans le sens de lecture\n- (..) : Habitat suspecté.';
COMMENT ON COLUMN ps.v_pelouseseches_all.lb_hab2 IS 'Libellé(s) CORINE Biotopes constituant(s) l''habitat 2 sur 4 de la pelouse sèche. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN ps.v_pelouseseches_all.n_hab3 IS 'Pourcentage de présence de l''habitat tertiaire sur la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_all.cb_hab3 IS 'Identifiants CORINE Biotopes constituant l''habitat 3 sur 4 de la pelouse sèche.\n- & : Entremêlement d''habitat\n- / : Habitat en transition dans le sens de lecture\n- (..) : Habitat suspecté.';
COMMENT ON COLUMN ps.v_pelouseseches_all.lb_hab3 IS 'Libellé(s) CORINE Biotopes constituant(s) l''habitat 3 sur 4 de la pelouse sèche. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN ps.v_pelouseseches_all.n_hab4 IS 'Pourcentage de présence de l''habitat majoritaire sur la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_all.cb_hab4 IS 'Identifiants CORINE Biotopes constituant l''habitat 4 sur 4 de la pelouse sèche.\n- & : Entremêlement d''habitat\n- / : Habitat en transition dans le sens de lecture\n- (..) : Habitat suspecté.';
COMMENT ON COLUMN ps.v_pelouseseches_all.lb_hab4 IS 'Libellé(s) CORINE Biotopes constituant(s) l''habitat 4 sur 4 de la pelouse sèche. Chaque élément de la liste est séparé par un '';''.';
"""
with con.begin() as cnx:
cnx.execute(v_pelouseseches_all)
cnx.execute(grant)
cnx.execute(comment)
v_termophile = (v_pelouseseches_all
.replace('v_pelouseseches_all','v_termophile')
.replace('Pelouses sèches','Milieux thermophile'))
grant = grant.replace('v_pelouseseches_all','v_termophile')
comment = (comment
.replace('v_pelouseseches_all','v_termophile')
.replace('de la pelouse sèche','du milieux thermophile'))
with con.begin() as cnx:
cnx.execute(v_termophile)
cnx.execute(grant)
cnx.execute(comment)
v_pelouseseches = """
CREATE OR REPLACE VIEW ps.v_pelouseseches
AS
SELECT
*
FROM ps.v_pelouseseches_all s
LEFT JOIN ps.r_infeq_1200m s7 ON s.site_code = s7.id_site
WHERE
s7.infeq_1200 IS True
ORDER BY s.site_code DESC NULLS LAST
"""
grant = """
GRANT ALL ON TABLE ps.v_pelouseseches TO grp_admin;
GRANT SELECT ON TABLE ps.v_pelouseseches TO grp_consult;
"""
comment = """
COMMENT ON VIEW ps.v_pelouseseches IS 'Vue des sites à pelouses sèches strictes, situées à niveau ou sous 1200 mètre';
COMMENT ON COLUMN ps.v_pelouseseches.site_code IS 'Identifiant de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches.nom IS 'Nom de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches.district_nat IS 'District naturel sur lequel est positionner la majeur partie de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches.source IS 'Nom de la couche source de la pelouse sèche au sein du CEN Isère.';
COMMENT ON COLUMN ps.v_pelouseseches.id_origine IS 'Identifiant d''origine de la pelouse sèche au sein de la couche source.';
COMMENT ON COLUMN ps.v_pelouseseches.auteur_site IS 'Auteur ayant caractérisé la pelouse sèche pour la première fois.';
COMMENT ON COLUMN ps.v_pelouseseches.auteur_geom IS 'Auteur ayant définis la géometrie actuelle de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches.auteur_attrs IS 'Auteur le plus récent ayant défini les attributs de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches.date_site IS 'Date de la première description de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches.date_geom IS 'Date de la géométrie.';
COMMENT ON COLUMN ps.v_pelouseseches.date_attrs IS 'Date des attributs les plus récents.';
COMMENT ON COLUMN ps.v_pelouseseches.type_milieu IS 'Caractérisation du milieu.';
COMMENT ON COLUMN ps.v_pelouseseches.type_site IS '';
COMMENT ON COLUMN ps.v_pelouseseches.rmq_site IS 'Remarques générale concernant la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches.rmq_fct_majeur IS 'Remarques sur les fonctions majeurs de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches.rmq_interet_patri IS 'Remarques sur les intérêts patrimoniaux de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches.rmq_bilan_menace IS 'Remarques globales sur les menaces qui concernent la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches.rmq_orient_act IS 'Remarques sur les orientations et les actes de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches.rmq_usage_process IS 'Remarques concernant les usages et les processus naturels de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches.statut IS 'Statut (Communautaire, Prioritaire ou non défini) de la pelouse sèche définit suivant la Directive Habitat.';
COMMENT ON COLUMN ps.v_pelouseseches.pratique IS 'Pratique agricole réalisée sur la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches.recouvrement IS 'Niveau de recouvrement du sol herbacé par rapport au sol nu.';
COMMENT ON COLUMN ps.v_pelouseseches.embrouss IS 'Niveau d''embroussaillement de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches.tx_embrouss IS 'Taux d''embroussaillement de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches.n_hab1 IS 'Pourcentage de présence de l''habitat 1 sur 4 de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches.cb_hab1 IS 'Identifiants CORINE Biotopes constituant l''habitat 1 sur 4 de la pelouse sèche.\n- & : Entremêlement d''habitat\n- / : Habitat en transition dans le sens de lecture\n- (..) : Habitat suspecté.';
COMMENT ON COLUMN ps.v_pelouseseches.lb_hab1 IS 'Libellé(s) CORINE Biotopes constituant(s) l''habitat 1 sur 4 de la pelouse sèche. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN ps.v_pelouseseches.n_hab2 IS 'Pourcentage de présence de l''habitat 2 sur 4 de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches.cb_hab2 IS 'Identifiants CORINE Biotopes constituant l''habitat 2 sur 4 de la pelouse sèche.\n- & : Entremêlement d''habitat\n- / : Habitat en transition dans le sens de lecture\n- (..) : Habitat suspecté.';
COMMENT ON COLUMN ps.v_pelouseseches.lb_hab2 IS 'Libellé(s) CORINE Biotopes constituant(s) l''habitat 2 sur 4 de la pelouse sèche. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN ps.v_pelouseseches.n_hab3 IS 'Pourcentage de présence de l''habitat tertiaire sur la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches.cb_hab3 IS 'Identifiants CORINE Biotopes constituant l''habitat 3 sur 4 de la pelouse sèche.\n- & : Entremêlement d''habitat\n- / : Habitat en transition dans le sens de lecture\n- (..) : Habitat suspecté.';
COMMENT ON COLUMN ps.v_pelouseseches.lb_hab3 IS 'Libellé(s) CORINE Biotopes constituant(s) l''habitat 3 sur 4 de la pelouse sèche. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN ps.v_pelouseseches.n_hab4 IS 'Pourcentage de présence de l''habitat majoritaire sur la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches.cb_hab4 IS 'Identifiants CORINE Biotopes constituant l''habitat 4 sur 4 de la pelouse sèche.\n- & : Entremêlement d''habitat\n- / : Habitat en transition dans le sens de lecture\n- (..) : Habitat suspecté.';
COMMENT ON COLUMN ps.v_pelouseseches.lb_hab4 IS 'Libellé(s) CORINE Biotopes constituant(s) l''habitat 4 sur 4 de la pelouse sèche. Chaque élément de la liste est séparé par un '';''.';
"""
with con.begin() as cnx:
cnx.execute(v_pelouseseches)
cnx.execute(grant)
cnx.execute(comment)
v_pelouseseches_sup1200 = """
CREATE OR REPLACE VIEW ps.v_pelouseseches_sup1200
AS
SELECT
*
FROM ps.v_pelouseseches_all s
LEFT JOIN ps.r_infeq_1200m s7 ON s.site_code = s7.id_site
WHERE
s7.infeq_1200 IS False
ORDER BY s.site_code DESC NULLS LAST
"""
grant = """
GRANT ALL ON TABLE ps.v_pelouseseches_sup1200 TO grp_admin;
GRANT SELECT ON TABLE ps.v_pelouseseches_sup1200 TO grp_consult;
"""
comment = """
COMMENT ON VIEW ps.v_pelouseseches IS 'Vue des sites à pelouses sèches strictes, situées au dessus de 1200 mètre';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.site_code IS 'Identifiant de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.nom IS 'Nom de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.district_nat IS 'District naturel sur lequel est positionner la majeur partie de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.source IS 'Nom de la couche source de la pelouse sèche au sein du CEN Isère.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.id_origine IS 'Identifiant d''origine de la pelouse sèche au sein de la couche source.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.auteur_site IS 'Auteur ayant caractérisé la pelouse sèche pour la première fois.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.auteur_geom IS 'Auteur ayant définis la géometrie actuelle de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.auteur_attrs IS 'Auteur le plus récent ayant défini les attributs de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.date_site IS 'Date de la première description de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.date_geom IS 'Date de la géométrie.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.date_attrs IS 'Date des attributs les plus récents.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.type_milieu IS 'Caractérisation du milieu.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.type_site IS '';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.rmq_site IS 'Remarques générale concernant la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.rmq_fct_majeur IS 'Remarques sur les fonctions majeurs de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.rmq_interet_patri IS 'Remarques sur les intérêts patrimoniaux de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.rmq_bilan_menace IS 'Remarques globales sur les menaces qui concernent la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.rmq_orient_act IS 'Remarques sur les orientations et les actes de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.rmq_usage_process IS 'Remarques concernant les usages et les processus naturels de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.statut IS 'Statut (Communautaire, Prioritaire ou non défini) de la pelouse sèche définit suivant la Directive Habitat.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.pratique IS 'Pratique agricole réalisée sur la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.recouvrement IS 'Niveau de recouvrement du sol herbacé par rapport au sol nu.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.embrouss IS 'Niveau d''embroussaillement de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.tx_embrouss IS 'Taux d''embroussaillement de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.n_hab1 IS 'Pourcentage de présence de l''habitat 1 sur 4 de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.cb_hab1 IS 'Identifiants CORINE Biotopes constituant l''habitat 1 sur 4 de la pelouse sèche.\n- & : Entremêlement d''habitat\n- / : Habitat en transition dans le sens de lecture\n- (..) : Habitat suspecté.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.lb_hab1 IS 'Libellé(s) CORINE Biotopes constituant(s) l''habitat 1 sur 4 de la pelouse sèche. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.n_hab2 IS 'Pourcentage de présence de l''habitat 2 sur 4 de la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.cb_hab2 IS 'Identifiants CORINE Biotopes constituant l''habitat 2 sur 4 de la pelouse sèche.\n- & : Entremêlement d''habitat\n- / : Habitat en transition dans le sens de lecture\n- (..) : Habitat suspecté.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.lb_hab2 IS 'Libellé(s) CORINE Biotopes constituant(s) l''habitat 2 sur 4 de la pelouse sèche. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.n_hab3 IS 'Pourcentage de présence de l''habitat tertiaire sur la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.cb_hab3 IS 'Identifiants CORINE Biotopes constituant l''habitat 3 sur 4 de la pelouse sèche.\n- & : Entremêlement d''habitat\n- / : Habitat en transition dans le sens de lecture\n- (..) : Habitat suspecté.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.lb_hab3 IS 'Libellé(s) CORINE Biotopes constituant(s) l''habitat 3 sur 4 de la pelouse sèche. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.n_hab4 IS 'Pourcentage de présence de l''habitat majoritaire sur la pelouse sèche.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.cb_hab4 IS 'Identifiants CORINE Biotopes constituant l''habitat 4 sur 4 de la pelouse sèche.\n- & : Entremêlement d''habitat\n- / : Habitat en transition dans le sens de lecture\n- (..) : Habitat suspecté.';
COMMENT ON COLUMN ps.v_pelouseseches_sup1200.lb_hab4 IS 'Libellé(s) CORINE Biotopes constituant(s) l''habitat 4 sur 4 de la pelouse sèche. Chaque élément de la liste est séparé par un '';''.';
"""
with con.begin() as cnx:
cnx.execute(v_pelouseseches_sup1200)
cnx.execute(grant)
cnx.execute(comment)
v_pelouseseches_noalti = """
CREATE OR REPLACE VIEW ps.v_pelouseseches_noalti
AS
SELECT
*
FROM ps.v_pelouseseches_all s
LEFT JOIN ps.r_infeq_1200m s7 ON s.site_code = s7.id_site
WHERE
s7.infeq_1200 IS NULL
ORDER BY s.site_code DESC NULLS LAST
"""
grant = """
GRANT ALL ON TABLE ps.v_pelouseseches_noalti TO grp_admin;
"""
comment = """
COMMENT ON VIEW ps.v_pelouseseches_noalti IS 'Vue des sites à pelouses sèches strictes, ne possèdant pas de critères ''infeq_1200''';
"""
with con.begin() as cnx:
cnx.execute(v_pelouseseches_noalti)
cnx.execute(grant)
cnx.execute(comment)
# test = """
# SELECT * FROM ps.v_pelouseseches_all;
# """
# data = gpd.read_postgis(
# sql = test,
# con = con)
3_AZALEE/create_view_ref.py
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-.
from sqlalchemy import create_engine, text
from sqlalchemy.engine import URL
from datetime import datetime as dt
import pandas as pd
import geopandas as gpd
# Parametres bdd
from pycen import con
v_rpg2021_ilots_anonymes_isere = """
DROP VIEW IF EXISTS ref_territoire.v_rpg2021_ilots_anonymes_isere;
CREATE OR REPLACE VIEW ref_territoire.v_rpg2021_ilots_anonymes_isere
AS SELECT --s.id,
s.*
FROM ref_territoire.rpg2021_ilots_anonymes_reg s,
ref_territoire.dept_isere reg
WHERE st_intersects(s.geom, reg.geom);
"""
grant_rpg2021_ilots = """
ALTER TABLE ref_territoire.v_rpg2021_ilots_anonymes_isere OWNER TO cen_admin;
GRANT ALL ON TABLE ref_territoire.v_rpg2021_ilots_anonymes_isere TO cen_admin;
GRANT ALL ON TABLE ref_territoire.v_rpg2021_ilots_anonymes_isere TO grp_admin;
GRANT SELECT ON TABLE ref_territoire.v_rpg2021_ilots_anonymes_isere TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(v_rpg2021_ilots_anonymes_isere)
cnx.execute(grant_rpg2021_ilots)
v_rpg2021_parcelles_graphiques_isere = (v_rpg2021_ilots_anonymes_isere
.replace('v_rpg2021_ilots_anonymes_isere','v_rpg2021_parcelles_graphiques_isere')
.replace('rpg2021_ilots_anonymes_reg','rpg2021_parcelles_graphiques_reg'))
grant = grant_rpg2021_ilots.replace('v_rpg2021_ilots_anonymes_isere','v_rpg2021_parcelles_graphiques_isere')
with con.begin() as cnx:
cnx.execute(v_rpg2021_parcelles_graphiques_isere)
cnx.execute(grant)
v_rpg2020_ilots_anonymes_isere = v_rpg2021_ilots_anonymes_isere.replace('2021','2020')
grant = grant_rpg2021_ilots.replace('2021','2020')
with con.begin() as cnx:
cnx.execute(v_rpg2020_ilots_anonymes_isere)
cnx.execute(grant)
v_rpg2020_parcelles_graphiques_isere = v_rpg2021_ilots_anonymes_isere.replace('v_rpg2021_ilots_anonymes_isere','v_rpg2020_parcelles_graphiques_isere')
grant = grant_rpg2021_ilots.replace('v_rpg2021_ilots_anonymes_isere','v_rpg2020_parcelles_graphiques_isere')
with con.begin() as cnx:
cnx.execute(v_rpg2020_parcelles_graphiques_isere)
cnx.execute(grant)
3_AZALEE/create_view_site.py
+76
View File
@@ -0,0 +1,76 @@
from pycen import con
from sqlalchemy import text
v_sites = '''
DROP VIEW IF EXISTS sites.v_sites;
CREATE OR REPLACE VIEW sites.v_sites
AS
WITH tmp_auteur_site as (
SELECT
s.id,
string_agg(s11.auteur,';' ORDER BY s11.auteur) AS auteur_site
FROM sites.sites s
JOIN sites.type_milieu s3 ON s.id_type_milieu = s3.id
LEFT JOIN (sites.r_sites_auteur s1
JOIN personnes.v_personne s11 ON s1.id_auteur = s11.id
) ON s.id::text = s1.id_site::text
WHERE s.date_fin IS NULL
GROUP BY s.id
ORDER BY s.id
), tmp_auteur_geom as (
SELECT
s6.id,
string_agg(s611.auteur,';' ORDER BY s611.auteur) AS auteur_geom
FROM sites.sites s
JOIN sites.type_milieu s3 ON s.id_type_milieu = s3.id
LEFT JOIN (sites.r_sites_geom s6
LEFT JOIN (sites.r_geomsites_auteur s61
JOIN personnes.v_personne s611 ON s61.id_auteur = s611.id
) ON s6.id = s61.id_geom_site
) ON s.id = s6.id_site
WHERE s.date_fin IS NULL
GROUP BY s6.id
ORDER BY s6.id
)
SELECT
g.id id_geom_site,
s.id code_site,
s.nom,
s1.auteur_site,
g1.auteur_geom,
s.date_deb date,
s.date_fin,
s.id_old_site,
s3.nom_court AS type_milieu,
s4.nom AS type_site,
s5.id||' - '||s5.nom AS typo_sdage,
s.dept,
s.org,
s.num,
s.remarques,
s.nom_com,
s.nom_com_json,
g.geom,
g.date date_geom,
g.link_pdf,
g.rmq_fct_majeur,
g.rmq_interet_patri,
g.rmq_bilan_menace,
g.rmq_orient_act,
g.rmq_usage_process,
g2.libelle lot,
g.id_origine,
g.date_insert
FROM sites.sites s
LEFT JOIN tmp_auteur_site s1 ON s.id = s1.id
LEFT JOIN sites.autre_nom s2 ON s.id = s2.id_site
LEFT JOIN sites.type_milieu s3 ON s.id_type_milieu = s3.id
LEFT JOIN sites.type_site s4 ON s.id_type_site::text = s4.id::text
LEFT JOIN sites.typo_sdage s5 ON s.id_typo_sdage::text = s5.id::text
LEFT JOIN sites.r_sites_geom g ON s.id = g.id_site
LEFT JOIN tmp_auteur_geom g1 ON g.id = g1.id
LEFT JOIN sites.lots g2 USING (id_lot)
;
'''
with con.begin() as cnx:
cnx.execute(text(v_sites))
3_AZALEE/create_view_zh.py
+555
View File
@@ -0,0 +1,555 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-.
from sqlalchemy import create_engine, text
from sqlalchemy.engine import URL
from datetime import datetime as dt
import pandas as pd
import geopandas as gpd
# Parametres bdd
from pycen import con
drop_v_zh = 'DROP VIEW IF EXISTS zones_humides.v_zoneshumides CASCADE;'
with con.begin() as cnx:
cnx.execute(drop_v_zh)
v_zh_hab = """
DROP VIEW IF EXISTS zones_humides.v_zh_hab;
CREATE OR REPLACE VIEW zones_humides.v_zh_hab
AS
WITH auteur AS (
SELECT DISTINCT ON (id_sitehab)
c.id_sitehab,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM zones_humides.r_rsitehab_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_sitehab
ORDER BY 1
)
SELECT DISTINCT ON (a.id_geom_site)
a.id_geom_site,
c.auteur,
MAX(a.date) date,
string_agg(a.id_cb,';') code_cb,
string_agg(b.lb_hab_fr,';') lib_cb
FROM zones_humides.r_site_habitat a
JOIN ref_habitats.corine_biotope b ON a.id_cb = b.id
JOIN auteur c ON c.id_sitehab = a.id
WHERE a."valid"
GROUP BY 1,2
ORDER BY a.id_geom_site
"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zh_hab TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zh_hab TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(text(v_zh_hab))
cnx.execute(grant)
v_zh_usgprocess = """
DROP VIEW IF EXISTS zones_humides.v_zh_usgprocess;
CREATE OR REPLACE VIEW zones_humides.v_zh_usgprocess
AS
WITH auteur AS (
SELECT DISTINCT ON (id_siteusage)
c.id_siteusage,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM zones_humides.r_rsiteusage_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_siteusage
ORDER BY 1
), usgprocess as (
SELECT distinct on (a.id_geom_site,a1.nom,a2.nom)
a.id_geom_site,
c.auteur,
MAX(a.date) date,
a1.nom activ_hum,
a.activ_hum_autre,
a2.nom impact,
a3.nom "position",
a.remarques rmq_activ_hum
FROM zones_humides.r_site_usageprocess a
JOIN zones_humides.param_activ_hum a1 ON a.id_activ_hum = a1.id
JOIN zones_humides.param_impact a2 ON a.id_impact = a2.id
JOIN zones_humides.param_position a3 ON a.id_position = a3.id
JOIN auteur c ON c.id_siteusage = a.id
WHERE a."valid"
--AND date = a.date
GROUP BY a.id_geom_site,c.auteur,a1.nom,a.activ_hum_autre,a2.nom,a3.nom,a.remarques
ORDER BY 1
)
SELECT DISTINCT ON (id_geom_site)
id_geom_site,
auteur,
MAX(date) date,
string_agg(
CASE WHEN activ_hum = 'Autre (préciser dans l''encart réservé aux remarques)' THEN activ_hum_autre
ELSE activ_hum END,';') activ_hum,
string_agg(
CASE WHEN impact = 'Autre (préciser dans l''encart réservé aux remarques)' THEN rmq_activ_hum
ELSE impact END,';') impact,
string_agg("position",';') "position",
string_agg(
CASE WHEN impact = 'Autre (préciser dans l''encart réservé aux remarques)' THEN NULL
ELSE rmq_activ_hum END,';') rmq_activ_hum
FROM usgprocess
GROUP BY 1,2
ORDER BY 1
"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zh_usgprocess TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zh_usgprocess TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(v_zh_usgprocess)
cnx.execute(grant)
v_zh_connex = """
DROP VIEW IF EXISTS zones_humides.v_zh_connex;
CREATE OR REPLACE VIEW zones_humides.v_zh_connex
AS
WITH auteur AS (
SELECT DISTINCT ON (id_siteconnect)
c.id_siteconnect,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM zones_humides.r_rsiteconnect_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_siteconnect
ORDER BY 1
)
SELECT
a.id_geom_site,
c.auteur,
a.date,
b.nom connexion
FROM zones_humides.r_site_type_connect a
JOIN zones_humides.param_type_connect b ON a.id_param_connect = b.id
JOIN auteur c ON c.id_siteconnect = a.id
WHERE a."valid"
AND a.date = (SELECT MAX(date) FROM zones_humides.r_site_type_connect WHERE id_geom_site = a.id_geom_site)
ORDER BY 1
"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zh_connex TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zh_connex TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(v_zh_connex)
cnx.execute(grant)
v_zh_submertion = """
DROP VIEW IF EXISTS zones_humides.v_zh_submertion;
CREATE OR REPLACE VIEW zones_humides.v_zh_submertion
AS
WITH auteur AS (
SELECT DISTINCT ON (id_sitesub)
c.id_sitesub,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM zones_humides.r_rsitesub_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_sitesub
ORDER BY 1
)
SELECT
a.id_geom_site,
c.auteur,
a.date,
a1.nom subm_freq,
a2.nom subm_etend,
a.id_origsub subm_orig
FROM zones_humides.r_site_sub a
JOIN (zones_humides.param_sub a1 JOIN zones_humides.type_param_sub s6411 ON s6411.id = a1.id_type) ON a.id_freqsub = a1.id
JOIN (zones_humides.param_sub a2 JOIN zones_humides.type_param_sub s6421 ON s6421.id = a2.id_type) ON a.id_etendsub = a2.id
JOIN auteur c ON c.id_sitesub = a.id
WHERE a."valid"
AND a.date = (SELECT MAX(date) FROM zones_humides.r_site_sub WHERE id_geom_site = a.id_geom_site)
AND s6411.nom = 'Submersion fréquente'
AND s6421.nom = 'Submersion étendue'
ORDER BY 1
"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zh_submertion TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zh_submertion TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(v_zh_submertion)
cnx.execute(grant)
v_zh_fctecosociopatri_cross = """
DROP VIEW IF EXISTS zones_humides.v_zh_fctecosociopatri_cross;
CREATE OR REPLACE VIEW zones_humides.v_zh_fctecosociopatri_cross
AS
SELECT
id_geom_site,
auteur,
MAX(date) date,
fct_bio,
fct_hydro,
int_patri,
"val_socioEco"
FROM crosstab(
$$WITH auteur AS (
SELECT DISTINCT ON (id_sitefct)
c.id_sitefct,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM zones_humides.r_rsitefct_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_sitefct
ORDER BY 1
)
SELECT
a.id_geom_site::bigint,
d.auteur,
MAX(a.date) date,
c.nom_court type_param,
string_agg(
CASE WHEN a.description IS NULL THEN b.nom
WHEN a.description = '' THEN b.nom
ELSE CONCAT(b.nom,' (',a.description,')') END,
';') fct
FROM zones_humides.r_site_fctecosociopatri a
JOIN (zones_humides.param_fct_eco_socio_patri b
JOIN zones_humides.type_param_fct c ON b.id_type = c.id
) ON a.id_fct = b.id
JOIN auteur d ON d.id_sitefct = a.id
WHERE a."valid"
GROUP BY a.id_geom_site,d.auteur,c.nom_court
ORDER BY 1,4;$$,
$$SELECT DISTINCT nom_court FROM zones_humides.type_param_fct ORDER BY 1 ASC;$$
) AS ct (
"id_geom_site" bigint,
"auteur" text,
"date" date,
"fct_bio" text,
"fct_hydro" text,
"int_patri" text,
"val_socioEco" text)
GROUP BY id_geom_site,auteur,fct_bio,fct_hydro,int_patri,"val_socioEco"
ORDER BY 1,3,2;
"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zh_fctecosociopatri_cross TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zh_fctecosociopatri_cross TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(v_zh_fctecosociopatri_cross)
cnx.execute(grant)
v_zh_critdelim_cross = """
DROP VIEW IF EXISTS zones_humides.v_zh_critdelim_cross;
CREATE OR REPLACE VIEW zones_humides.v_zh_critdelim_cross
AS
SELECT DISTINCT ON (id_geom_site)
id_geom_site,
auteur,
MAX(date) date,
crit_delim,
crit_def_esp
FROM crosstab(
$$WITH auteur AS (
SELECT DISTINCT ON (id_sitedelim)
c.id_sitedelim,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM zones_humides.r_rsitedelim_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_sitedelim
ORDER BY 1
)
SELECT
a.id_geom_site::bigint,
d.auteur,
MAX(a.date) date,
c.nom_court type_param,
string_agg(
CASE WHEN a.description IS NULL THEN b.nom
WHEN a.description = '' THEN b.nom
ELSE CONCAT(b.nom,' (',a.description,')') END,
';') fct
FROM zones_humides.r_site_critdelim a
JOIN (zones_humides.param_delim_fct b
JOIN zones_humides.type_param_delim_fct c ON b.id_type = c.id
) ON a.id_crit_delim = b.id
JOIN auteur d ON d.id_sitedelim = a.id
WHERE a."valid"
GROUP BY a.id_geom_site,d.auteur,c.nom_court
ORDER BY 1,2,3$$,
$$SELECT DISTINCT nom_court FROM zones_humides.type_param_delim_fct ORDER BY 1 DESC;$$
) AS ct (
"id_geom_site" bigint,
"auteur" text,
"date" date,
"crit_delim" text,
"crit_def_esp" text)
GROUP BY id_geom_site,auteur,crit_delim,crit_def_esp
ORDER BY 1,3,2;
"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zh_critdelim_cross TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zh_critdelim_cross TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(v_zh_critdelim_cross)
cnx.execute(grant)
v_zh_reghydro_cross = """
DROP VIEW IF EXISTS zones_humides.v_zh_reghydro_cross;
CREATE OR REPLACE VIEW zones_humides.v_zh_reghydro_cross
AS
SELECT DISTINCT ON (id_geom_site)
id_geom_site,
auteur,
MAX(date) date,
(string_to_array(ct.entree_eau,'//'))[1] entree_eau_reg,
(string_to_array(ct.entree_eau,'//'))[2] entree_eau_perm,
(string_to_array(ct.entree_eau,'//'))[3] entree_eau_topo,
(string_to_array(ct.sortie_eau,'//'))[1] sortie_eau_reg,
(string_to_array(ct.sortie_eau,'//'))[2] sortie_eau_perm,
(string_to_array(ct.sortie_eau,'//'))[3] sortie_eau_topo
FROM crosstab(
$$WITH auteur AS (
SELECT DISTINCT ON (id_sitehydro)
c.id_sitehydro,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM zones_humides.r_rsitehydro_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_sitehydro
ORDER BY 1
)
SELECT
a.id_geom_site,
d.auteur,
MAX(a.date) date,
a.in_out,
CONCAT(
string_agg(a1.nom,';'),'//', -- reg_hydro
string_agg(a2.nom,';'),'//', -- permanence
string_agg(a.rmq_toponymie,';') -- rmq_toponymie
) hydro
FROM zones_humides.r_site_reghydro a
LEFT JOIN zones_humides.param_reg_hydro a1 ON a.id_reg_hydro = a1.id
LEFT JOIN zones_humides.param_permanence a2 ON a.id_permanence = a2.id
JOIN auteur d ON d.id_sitehydro = a.id
WHERE a."valid"
GROUP BY a.id_geom_site,d.auteur,a.in_out
ORDER BY 1,2$$,
$$SELECT DISTINCT in_out FROM zones_humides.r_site_reghydro ORDER BY 1 DESC;$$
) AS ct (
"id_geom_site" bigint,
"auteur" text,
"date" date,
"entree_eau" text,
"sortie_eau" text)
GROUP BY id_geom_site,auteur,entree_eau,sortie_eau
ORDER BY 1,3,2;
"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zh_reghydro_cross TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zh_reghydro_cross TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(v_zh_reghydro_cross)
cnx.execute(grant)
v_zoneshumides = """
DROP VIEW IF EXISTS zones_humides.v_zoneshumides CASCADE;
CREATE OR REPLACE VIEW zones_humides.v_zoneshumides
AS
WITH tmp_auteur_site as (
SELECT
s.id,
string_agg(s11.auteur,';' ORDER BY s11.auteur) AS auteur_site
FROM sites.sites s
JOIN sites.type_milieu s3 ON s.id_type_milieu = s3.id
LEFT JOIN (sites.r_sites_auteur s1
JOIN personnes.v_personne s11 ON s1.id_auteur = s11.id
) ON s.id::text = s1.id_site::text
WHERE s.date_fin IS NULL
AND s3.nom_court::text = 'Milieux humides'::text
GROUP BY s.id
ORDER BY s.id
), tmp_auteur_geom as (
SELECT
s6.id,
string_agg(s611.auteur,';' ORDER BY s611.auteur) AS auteur_geom
FROM sites.sites s
JOIN sites.type_milieu s3 ON s.id_type_milieu = s3.id
LEFT JOIN (sites.r_sites_geom s6
LEFT JOIN (sites.r_geomsites_auteur s61
JOIN personnes.v_personne s611 ON s61.id_auteur = s611.id
) ON s6.id = s61.id_geom_site
) ON s.id = s6.id_site
WHERE s.date_fin IS NULL
AND s3.nom_court::text = 'Milieux humides'::text
GROUP BY s6.id
ORDER BY s6.id
)
SELECT
s.id AS site_code,
(SELECT ARRAY_AGG(id_site_old) FROM (values (s7.id_site_old)) AS value(id_site_old) WHERE id_site_old IS NOT NULL) old_code,
s.nom,
s2.autre_nom,
s1.auteur_site,
s61.auteur_geom,
(
SELECT DISTINCT ON (date) auteur
FROM (values (s6.date,s61.auteur_geom),(s62.date,s62.auteur),(s63.date,s63.auteur),(s64.date,s64.auteur),(s65.date,s65.auteur),
(s66.date,s66.auteur),(s67.date,s67.auteur),(s68.date,s68.auteur)) AS value(date,auteur)
WHERE date = (SELECT MAX(date) FROM (values (s6.date),(s62.date),(s63.date),(s64.date),(s65.date),(s66.date),(s67.date),(s68.date)) AS value(date))
) as auteur_last_maj,
s.date_deb AS date_site,
s6.date AS date_geom,
(SELECT MAX(date) FROM (values (s6.date),(s62.date),(s63.date),(s64.date),(s65.date),(s66.date),(s67.date),(s68.date)) AS value(date)) as date_last_maj,
s3.nom_court AS type_milieu,
s4.nom AS type_site,
s5.id||' - '||s5.nom AS typo_sdage,
s5.mnemo mnemo_sdage,
s.remarques AS rmq_site,
s6.rmq_fct_majeur,
s6.rmq_interet_patri,
s6.rmq_bilan_menace,
s6.rmq_orient_act,
s6.rmq_usage_process,
s62.code_cb,
s62.lib_cb,
s63.activ_hum,
s63.impact,
s63."position",
s63.rmq_activ_hum,
s64.connexion,
s65.subm_orig,
s65.subm_freq,
s65.subm_etend,
s66.fct_bio,
s66.fct_hydro,
s66.int_patri,
s66."val_socioEco",
s67.crit_delim,
s67.crit_def_esp,
s68.entree_eau_reg,
s68.entree_eau_perm,
s68.entree_eau_topo,
s68.sortie_eau_reg,
s68.sortie_eau_perm,
s68.sortie_eau_topo,
s6.geom
FROM sites.sites s
LEFT JOIN tmp_auteur_site s1 ON s.id = s1.id
LEFT JOIN sites.autre_nom s2 ON s.id = s2.id_site
LEFT JOIN sites.type_milieu s3 ON s.id_type_milieu = s3.id
LEFT JOIN sites.type_site s4 ON s.id_type_site::text = s4.id::text
LEFT JOIN sites.typo_sdage s5 ON s.id_typo_sdage::text = s5.id::text
LEFT JOIN (sites.r_sites_geom s6
LEFT JOIN tmp_auteur_geom s61 ON s6.id = s61.id
LEFT JOIN zones_humides.v_zh_hab s62 ON s6.id = s62.id_geom_site
LEFT JOIN zones_humides.v_zh_usgprocess s63 ON s6.id = s63.id_geom_site
LEFT JOIN zones_humides.v_zh_connex s64 ON s6.id = s64.id_geom_site
LEFT JOIN zones_humides.v_zh_submertion s65 ON s6.id = s65.id_geom_site
LEFT JOIN zones_humides.v_zh_fctecosociopatri_cross s66 ON s6.id = s66.id_geom_site
LEFT JOIN zones_humides.v_zh_critdelim_cross s67 ON s6.id = s67.id_geom_site
LEFT JOIN zones_humides.v_zh_reghydro_cross s68 ON s6.id = s68.id_geom_site
) ON s.id::text = s6.id_site::text
LEFT JOIN sites.hist_id_site s7 ON s.id = s7.id_site_new
WHERE s.date_fin IS NULL
AND s3.nom_court::text = 'Milieux humides'::text
AND s6.date = (SELECT MAX(date) FROM sites.r_sites_geom WHERE id_site = s.id)
ORDER BY s.id ASC NULLS FIRST;
"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zoneshumides TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zoneshumides TO grp_consult;
"""
comment = """
COMMENT ON COLUMN zones_humides.v_zoneshumides.site_code IS 'Identifiant de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.nom IS 'Nom de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.auteur_site IS 'Auteur ayant caractérisé la zone humide pour la première fois.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.auteur_geom IS 'Auteur ayant définis la géometrie actuelle de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.auteur_last_maj IS 'Auteur le plus récent ayant défini les attributs de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.date_site IS 'Date de la première description de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.date_geom IS 'Date de la géométrie.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.date_last_maj IS 'Date des attributs les plus récents.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.type_milieu IS 'Caractérisation du milieu.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.type_site IS '';
COMMENT ON COLUMN zones_humides.v_zoneshumides.typo_sdage IS 'Typologie sdage de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_site IS 'Remarques générale concernant la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_fct_majeur IS 'Remarques sur les fonctions majeurs de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_interet_patri IS 'Remarques sur les intérêts patrimoniaux de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_bilan_menace IS 'Remarques globales sur les menaces qui concernent la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_orient_act IS 'Remarques sur les orientations et les actes de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_usage_process IS 'Remarques concernant les usages et les processus naturels de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.code_cb IS 'Liste des codes CORINE Biotopes identifiés sur la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.lib_cb IS 'Liste des habitats CORINE Biotopes identifiés sur la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.activ_hum IS 'Liste des activités humaines identifiées sur la zone humide. Chaque élément de la liste est séparé par un '';'' et est lié à la colonne ''impacts'' et ''position''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.impact IS 'Liste des impacts identifiés et engendrés par les activités humaine sur la zone humide. Chaque élément de la liste est séparé par un '';'' et est lié à la colonne ''activ_hum'' et ''position''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides."position" IS 'Liste des potionnements des différentes activités humaines sur la zone humide. Chaque élément de la liste est séparé par un '';'' et est lié à la colonne ''activ_hum'' et ''impact''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_activ_hum IS 'Remarques concernant activités humaines identifiées sur la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.connexion IS 'Connexion de la zone humide dans son environnement.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.subm_orig IS 'Origine de l''eau submerssive.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.subm_freq IS 'Fréquence de la submersion de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.subm_etend IS 'Etendue de la submersion de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.fct_bio IS 'Liste des fonctions biologiques de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.fct_hydro IS 'Liste des fonctions hydrologiques de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.int_patri IS 'Liste des interêts patrimoniaux de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides."val_socioEco" IS 'Liste des valeurs socio-economiques de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.crit_delim IS 'Liste des critères de délimitation de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.crit_def_esp IS 'Liste des critères de l''espace de fonctionnalité de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.entree_eau_reg IS 'Liste des entrées d''eau du régime hydrique de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.entree_eau_perm IS 'Liste des permanances respectivement de chaque entrée d''eau de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.entree_eau_topo IS 'Liste de la toponymie respectivement de chaque entrée d''eau de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.sortie_eau_reg IS 'Liste des sorties d''eau du régime hydrique de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.sortie_eau_perm IS 'Liste des permanances respectivement de chaque sortie d''eau de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.sortie_eau_topo IS 'Liste de la toponymie respectivement de chaque sortie d''eau de la zone humide. Chaque élément de la liste est séparé par un '';''.';
"""
with con.begin() as cnx:
cnx.execute(text(v_zoneshumides))
cnx.execute(grant)
cnx.execute(comment)
v_rhomeosite = """
DROP VIEW IF EXISTS zones_humides.v_rhomeosite;
CREATE OR REPLACE VIEW zones_humides.v_rhomeosite
AS
SELECT
v.site_code||' - '||v.nom "NAME",
SPLIT_PART(v.auteur_geom,' (',1) "REFERENT",
REPLACE(SPLIT_PART(v.auteur_geom,' (',2),')','') "ORG",
SPLIT_PART(v.typo_sdage,' - ',1) "TYPE",
CASE WHEN r.nom::text = 'alpin' THEN '1'
WHEN r.nom::text = 'continental' THEN '2'
WHEN r.nom::text = 'mediterraneen' THEN '4'
END "ODONATE",
v.geom
FROM zones_humides.v_zoneshumides v, ref_territoire.ref_biogeo r
WHERE st_intersects(v.geom, r.geom)
"""
grant = """
GRANT ALL ON TABLE zones_humides.v_rhomeosite TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_rhomeosite TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(text(v_rhomeosite))
cnx.execute(grant)
test = """
SELECT * FROM zones_humides.v_zoneshumides;
"""
data = pd.read_sql_query(
sql = text(test),
con = con)
data[~data.index.isin(data.drop_duplicates(subset='site_code',keep=False).index)]
data[data.duplicated(subset='site_code',keep=False)]
3_AZALEE/create_view_zh2.py
+594
View File
@@ -0,0 +1,594 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-.
from sqlalchemy import text
from datetime import datetime as dt
import geopandas as gpd
from pycen import con
drop_v_zh = 'DROP VIEW IF EXISTS zones_humides.v_zoneshumides CASCADE;'
with con.begin() as cnx:
cnx.execute(drop_v_zh)
v_zh_hab = """
DROP VIEW IF EXISTS zones_humides.v_zh_hab;
CREATE OR REPLACE VIEW zones_humides.v_zh_hab
AS
WITH auteur AS (
SELECT DISTINCT ON (id_sitehab)
c.id_sitehab,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM zones_humides.r_rsitehab_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_sitehab
ORDER BY 1
)
SELECT DISTINCT ON (a.id_site)
--a.id_geom_site,
a.id_site,
c.auteur,
--MAX(a."date") "date",
a."date",
string_agg(a.id_cb,';' order by a.id_cb) code_cb,
string_agg(b.lb_hab_fr,';' order by a.id_cb) lib_cb
FROM zones_humides.r_site_habitat a
JOIN ref_habitats.corine_biotope b ON a.id_cb = b.id
JOIN auteur c ON c.id_sitehab = a.id
WHERE a."valid"
GROUP BY 1,2,3
ORDER BY a.id_site, a."date" desc,row_number() OVER (ORDER BY a.id_site) desc
;"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zh_hab TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zh_hab TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(text(v_zh_hab))
cnx.execute(grant)
v_zh_usgprocess = """
DROP VIEW IF EXISTS zones_humides.v_zh_usgprocess;
CREATE OR REPLACE VIEW zones_humides.v_zh_usgprocess
AS
WITH auteur AS (
SELECT DISTINCT ON (id_siteusage)
c.id_siteusage,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM zones_humides.r_rsiteusage_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_siteusage
ORDER BY 1
), usgprocess as (
SELECT distinct on (a.id_site,a1.nom,a2.nom)
--a.id_geom_site,
a.id_site,
c.auteur,
a."date",
a1.nom activ_hum,
a.activ_hum_autre,
a2.nom impact,
a3.nom "position",
a.remarques rmq_activ_hum
FROM zones_humides.r_site_usageprocess a
JOIN zones_humides.param_activ_hum a1 ON a.id_activ_hum = a1.id
JOIN zones_humides.param_impact a2 ON a.id_impact = a2.id
JOIN zones_humides.param_position a3 ON a.id_position = a3.id
JOIN auteur c ON c.id_siteusage = a.id
WHERE a."valid"
--AND date = a.date
--GROUP BY a.id_geom_site,c.auteur,a1.nom,a.activ_hum_autre,a2.nom,a3.nom,a.remarques
--GROUP BY a.id_site,c.auteur,a1.nom,a.activ_hum_autre,a2.nom,a3.nom,a.remarques
ORDER BY a.id_site,a1.nom,a2.nom, a."date" desc,row_number() OVER (ORDER BY a.id_site) desc
)
SELECT DISTINCT ON (id_site)
--id_geom_site,
id_site,
auteur,
"date",
string_agg(
CASE WHEN activ_hum = 'Autre (préciser dans l''encart réservé aux remarques)' THEN activ_hum_autre
ELSE activ_hum END,';') activ_hum,
string_agg(
CASE WHEN impact = 'Autre (préciser dans l''encart réservé aux remarques)' THEN rmq_activ_hum
ELSE impact END,';') impact,
string_agg("position",';') "position",
string_agg(
CASE WHEN impact = 'Autre (préciser dans l''encart réservé aux remarques)' THEN NULL
ELSE rmq_activ_hum END,';') rmq_activ_hum
FROM usgprocess
GROUP BY 1,2,3
ORDER BY id_site, "date" desc,row_number() OVER (ORDER BY id_site) desc
;"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zh_usgprocess TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zh_usgprocess TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(v_zh_usgprocess)
cnx.execute(grant)
v_zh_connex = """
DROP VIEW IF EXISTS zones_humides.v_zh_connex;
CREATE OR REPLACE VIEW zones_humides.v_zh_connex
AS
WITH auteur AS (
SELECT DISTINCT ON (id_siteconnect)
c.id_siteconnect,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM zones_humides.r_rsiteconnect_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_siteconnect
ORDER BY 1
)
SELECT DISTINCT ON (id_site)
--a.id_geom_site,
a.id_site,
c.auteur,
a."date",
b.nom connexion
FROM zones_humides.r_site_type_connect a
JOIN zones_humides.param_type_connect b ON a.id_param_connect = b.id
JOIN auteur c ON c.id_siteconnect = a.id
WHERE a."valid"
--AND a.date = (SELECT MAX(date) FROM zones_humides.r_site_type_connect WHERE id_site = a.id_site)
ORDER BY a.id_site,"date" desc,row_number() OVER (ORDER BY a.id_site) desc
;"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zh_connex TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zh_connex TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(v_zh_connex)
cnx.execute(grant)
v_zh_submertion = """
DROP VIEW IF EXISTS zones_humides.v_zh_submertion;
CREATE OR REPLACE VIEW zones_humides.v_zh_submertion
AS
WITH auteur AS (
SELECT DISTINCT ON (id_sitesub)
c.id_sitesub,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM zones_humides.r_rsitesub_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_sitesub
ORDER BY 1
)
SELECT DISTINCT ON (a.id_site)
--a.id_geom_site,
a.id_site,
c.auteur,
a."date",
a1.nom subm_freq,
a2.nom subm_etend,
a.id_origsub subm_orig
FROM zones_humides.r_site_sub a
JOIN (zones_humides.param_sub a1 JOIN zones_humides.type_param_sub s6411 ON s6411.id = a1.id_type) ON a.id_freqsub = a1.id
JOIN (zones_humides.param_sub a2 JOIN zones_humides.type_param_sub s6421 ON s6421.id = a2.id_type) ON a.id_etendsub = a2.id
JOIN auteur c ON c.id_sitesub = a.id
WHERE a."valid"
--AND a.date = (SELECT MAX(date) FROM zones_humides.r_site_sub WHERE id_site = a.id_site)
AND s6411.nom = 'Submersion fréquente'
AND s6421.nom = 'Submersion étendue'
ORDER BY a.id_site,"date" desc,row_number() OVER (ORDER BY a.id_site) desc
;"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zh_submertion TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zh_submertion TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(v_zh_submertion)
cnx.execute(grant)
v_zh_fctecosociopatri_cross = """
DROP VIEW IF EXISTS zones_humides.v_zh_fctecosociopatri_cross;
CREATE OR REPLACE VIEW zones_humides.v_zh_fctecosociopatri_cross
AS
SELECT DISTINCT ON (id_site)
--id_geom_site,
id_site,
auteur,
"date",
fct_bio,
fct_hydro,
int_patri,
"val_socioEco"
FROM crosstab(
$$ WITH auteur AS (
SELECT DISTINCT ON (id_sitefct)
c.id_sitefct,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM zones_humides.r_rsitefct_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_sitefct
ORDER BY 1
), tmp_select as (
SELECT
id_site,
(SELECT regexp_split_to_table(auth,',') FROM (values (string_agg(d.auteur,',' order by ROW()))) AS value(auth) Limit 1) auteur,
MAX(a.date) date,
c.nom_court type_param,
b.nom,
string_agg(DISTINCT a.description,'\n') remark
FROM zones_humides.r_site_fctecosociopatri a
JOIN (zones_humides.param_fct_eco_socio_patri b
JOIN zones_humides.type_param_fct c ON b.id_type = c.id
) ON a.id_fct = b.id
JOIN auteur d ON d.id_sitefct = a.id
WHERE a."valid"
GROUP BY a.id_site,c.nom_court,b.nom
ORDER BY a.id_site,c.nom_court DESC,b.nom
)
SELECT
id_site,
string_agg(
DISTINCT CASE WHEN extract(year from date)>'1'
THEN auteur||'-'|| extract(year from date)
ELSE auteur END,
'; ') auteur,
MAX(date) date,
type_param,
string_agg(distinct
CASE WHEN remark IS NULL THEN nom
WHEN remark = '' THEN nom
ELSE CONCAT(nom,' (',remark,')') END,
';') fct
FROM tmp_select
GROUP BY 1,4
ORDER BY 1,4,3;$$,
$$SELECT DISTINCT nom_court FROM zones_humides.type_param_fct ORDER BY 1 ASC;$$
) AS ct (
"id_site" text,
"auteur" text,
"date" date,
"fct_bio" text,
"fct_hydro" text,
"int_patri" text,
"val_socioEco" text)
--GROUP BY id_site,auteur,fct_bio,fct_hydro,int_patri,"val_socioEco"
ORDER BY id_site,"date" desc,row_number() OVER (ORDER BY id_site) desc
;"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zh_fctecosociopatri_cross TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zh_fctecosociopatri_cross TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(v_zh_fctecosociopatri_cross)
cnx.execute(grant)
v_zh_critdelim_cross = """
DROP VIEW IF EXISTS zones_humides.v_zh_critdelim_cross;
CREATE OR REPLACE VIEW zones_humides.v_zh_critdelim_cross
AS
SELECT DISTINCT ON (id_site)
--id_geom_site,
id_site,
auteur,
"date",
crit_delim,
crit_def_esp
FROM crosstab(
$$WITH auteur AS (
SELECT DISTINCT ON (id_sitedelim)
c.id_sitedelim,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM zones_humides.r_rsitedelim_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_sitedelim
ORDER BY 1
)
SELECT
--a.id_geom_site::bigint,
id_site,
string_agg(
DISTINCT CASE WHEN extract(year from a.date)>'1'
THEN d.auteur||'-'|| extract(year from a.date)
ELSE d.auteur END,
'; ') auteur,
MAX(a.date) date,
c.nom_court type_param,
string_agg( DISTINCT
CASE WHEN a.description IS NULL THEN b.nom
WHEN a.description = '' THEN b.nom
ELSE CONCAT(b.nom,' (',a.description,')') END,
'; ') fct
FROM zones_humides.r_site_critdelim a
JOIN (zones_humides.param_delim_fct b
JOIN zones_humides.type_param_delim_fct c ON b.id_type = c.id
) ON a.id_crit_delim = b.id
JOIN auteur d ON d.id_sitedelim = a.id
WHERE a."valid"
GROUP BY 1,4
ORDER BY 3,1,2$$,
$$SELECT DISTINCT nom_court FROM zones_humides.type_param_delim_fct ORDER BY 1 DESC;$$
) AS ct (
"id_site" text,
"auteur" text,
"date" date,
"crit_delim" text,
"crit_def_esp" text)
--GROUP BY id_site,auteur,crit_delim,crit_def_esp
ORDER BY id_site,"date" desc,row_number() OVER (ORDER BY id_site) desc
;"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zh_critdelim_cross TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zh_critdelim_cross TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(v_zh_critdelim_cross)
cnx.execute(grant)
v_zh_reghydro_cross = """
DROP VIEW IF EXISTS zones_humides.v_zh_reghydro_cross;
CREATE OR REPLACE VIEW zones_humides.v_zh_reghydro_cross
AS
SELECT DISTINCT ON (id_site)
--id_geom_site,
id_site,
auteur,
"date",
(string_to_array(ct.entree_eau,'//'))[1] entree_eau_reg,
(string_to_array(ct.entree_eau,'//'))[2] entree_eau_perm,
(string_to_array(ct.entree_eau,'//'))[3] entree_eau_topo,
(string_to_array(ct.sortie_eau,'//'))[1] sortie_eau_reg,
(string_to_array(ct.sortie_eau,'//'))[2] sortie_eau_perm,
(string_to_array(ct.sortie_eau,'//'))[3] sortie_eau_topo
FROM crosstab(
$$WITH auteur AS (
SELECT DISTINCT ON (id_sitehydro)
c.id_sitehydro,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM zones_humides.r_rsitehydro_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_sitehydro
ORDER BY 1
)
SELECT
--a.id_geom_site,
id_site,
d.auteur,
MAX(a.date) date,
a.in_out,
CONCAT(
string_agg(a1.nom,';'),'//', -- reg_hydro
string_agg(a2.nom,';'),'//', -- permanence
string_agg(a.rmq_toponymie,';') -- rmq_toponymie
) hydro
FROM zones_humides.r_site_reghydro a
LEFT JOIN zones_humides.param_reg_hydro a1 ON a.id_reg_hydro = a1.id
LEFT JOIN zones_humides.param_permanence a2 ON a.id_permanence = a2.id
JOIN auteur d ON d.id_sitehydro = a.id
WHERE a."valid"
GROUP BY a.id_site,d.auteur,a.in_out
ORDER BY 1,2$$,
$$SELECT DISTINCT in_out FROM zones_humides.r_site_reghydro ORDER BY 1 DESC;$$
) AS ct (
"id_site" text,
"auteur" text,
"date" date,
"entree_eau" text,
"sortie_eau" text)
--GROUP BY id_site,auteur,entree_eau,sortie_eau
ORDER BY id_site,"date" desc,row_number() OVER (ORDER BY id_site) desc
;"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zh_reghydro_cross TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zh_reghydro_cross TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(v_zh_reghydro_cross)
cnx.execute(grant)
v_zoneshumides = """
DROP VIEW IF EXISTS zones_humides.v_zoneshumides CASCADE;
CREATE OR REPLACE VIEW zones_humides.v_zoneshumides
AS
WITH tmp_auteur_site as (
SELECT
s0.id,
string_agg(s11.auteur,';' ORDER BY s11.auteur) AS auteur_site
FROM sites.sites s0
JOIN sites.type_milieu s3 ON s0.id_type_milieu = s3.id
LEFT JOIN (sites.r_sites_auteur s1
JOIN personnes.v_personne s11 ON s1.id_auteur = s11.id
) ON s0.id::text = s1.id_site::text
WHERE s0.date_fin IS NULL
AND s3.nom_court::text = 'Milieux humides'::text
GROUP BY s0.id
ORDER BY s0.id
), tmp_auteur_geom as (
SELECT
s6.id,
string_agg(s611.auteur,';' ORDER BY s611.auteur) AS auteur_geom
FROM sites.sites s0
JOIN sites.type_milieu s3 ON s0.id_type_milieu = s3.id
LEFT JOIN (sites.r_sites_geom s6
LEFT JOIN (sites.r_geomsites_auteur s61
JOIN personnes.v_personne s611 ON s61.id_auteur = s611.id
) ON s6.id = s61.id_geom_site
) ON s0.id = s6.id_site
WHERE s0.date_fin IS NULL
AND s3.nom_court::text = 'Milieux humides'::text
GROUP BY s6.id
ORDER BY s6.id
),
tmp_other_name as (
SELECT
id_site_new,
ARRAY_AGG(id_site_old) id_site_old
FROM sites.r_site_maj
WHERE id_site_old IS NOT NULL
GROUP BY id_site_new
)
SELECT
s0.id AS site_code,
s7.id_site_old old_code,
s0.nom,
s2.autre_nom,
s1.auteur_site,
s6.auteur_geom,
(
SELECT DISTINCT ON (date) auteur
FROM (values (s6.date,s6.auteur_geom),(s62.date,s62.auteur),(s63.date,s63.auteur),(s64.date,s64.auteur),(s65.date,s65.auteur),
(s66.date,s66.auteur),(s67.date,s67.auteur),(s68.date,s68.auteur)) AS value(date,auteur)
WHERE date = (SELECT MAX(date) FROM (values (s6.date),(s62.date),(s63.date),(s64.date),(s65.date),(s66.date),(s67.date),(s68.date)) AS value(date))
) as auteur_last_maj,
s0.date_deb AS date_site,
s6.date AS date_geom,
(SELECT MAX(date) FROM (values (s6.date),(s62.date),(s63.date),(s64.date),(s65.date),(s66.date),(s67.date),(s68.date)) AS value(date)) as date_last_maj,
s3.nom_court AS type_milieu,
s4.nom AS type_site,
s5.id||' - '||s5.nom AS typo_sdage,
s5.mnemo mnemo_sdage,
s0.remarques AS rmq_site,
s6.rmq_fct_majeur,
s6.rmq_interet_patri,
s6.rmq_bilan_menace,
s6.rmq_orient_act,
s6.rmq_usage_process,
s62.code_cb,
s62.lib_cb,
s63.activ_hum,
s63.impact,
s63."position",
s63.rmq_activ_hum,
s64.connexion,
s65.subm_orig,
s65.subm_freq,
s65.subm_etend,
s66.fct_bio,
s66.fct_hydro,
s66.int_patri,
s66."val_socioEco",
s67.crit_delim,
s67.crit_def_esp,
s68.entree_eau_reg,
s68.entree_eau_perm,
s68.entree_eau_topo,
s68.sortie_eau_reg,
s68.sortie_eau_perm,
s68.sortie_eau_topo,
s6.id_origine,
s6.geom
FROM sites.sites s0
LEFT JOIN tmp_auteur_site s1 ON s0.id = s1.id
LEFT JOIN sites.autre_nom s2 ON s0.id = s2.id_site
LEFT JOIN sites.type_milieu s3 ON s0.id_type_milieu = s3.id
LEFT JOIN sites.type_site s4 ON s0.id_type_site::text = s4.id::text
LEFT JOIN sites.typo_sdage s5 ON s0.id_typo_sdage::text = s5.id::text
LEFT JOIN (SELECT DISTINCT ON (sg.id_site) *
FROM sites.r_sites_geom sg
LEFT JOIN tmp_auteur_geom s61 USING (id) --ON s6.id = s61.id
ORDER BY sg.id_site, "date" DESC, date_insert DESC
) s6 ON s0.id::text = s6.id_site::text
LEFT JOIN zones_humides.v_zh_hab s62 ON s0.id = s62.id_site
LEFT JOIN zones_humides.v_zh_usgprocess s63 ON s0.id = s63.id_site
LEFT JOIN zones_humides.v_zh_connex s64 ON s0.id = s64.id_site
LEFT JOIN zones_humides.v_zh_submertion s65 ON s0.id = s65.id_site
LEFT JOIN zones_humides.v_zh_fctecosociopatri_cross s66 ON s0.id = s66.id_site
LEFT JOIN zones_humides.v_zh_critdelim_cross s67 ON s0.id = s67.id_site
LEFT JOIN zones_humides.v_zh_reghydro_cross s68 ON s0.id = s68.id_site
LEFT JOIN tmp_other_name s7 ON s0.id = s7.id_site_new
WHERE s0.date_fin IS NULL
AND s3.nom_court::text = 'Milieux humides'::text
AND s6.date = (SELECT MAX(date) FROM sites.r_sites_geom WHERE r_sites_geom.id_site = s0.id)
ORDER BY s0.id ASC NULLS FIRST;
"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zoneshumides TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zoneshumides TO grp_consult;
"""
comment = """
COMMENT ON COLUMN zones_humides.v_zoneshumides.site_code IS 'Identifiant de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.old_code IS 'Ancien identifiant de la zone humide pouvant se retrouver dans des données historiques.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.nom IS 'Nom de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.autre_nom IS 'Autre nom de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.auteur_site IS 'Auteur ayant caractérisé la zone humide pour la première fois.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.auteur_geom IS 'Auteur ayant définis la géometrie actuelle de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.auteur_last_maj IS 'Auteur le plus récent ayant défini les attributs de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.date_site IS 'Date de la première description de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.date_geom IS 'Date de la géométrie.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.date_last_maj IS 'Date des attributs les plus récents.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.type_milieu IS 'Caractérisation du milieu.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.type_site IS '';
COMMENT ON COLUMN zones_humides.v_zoneshumides.typo_sdage IS 'Typologie sdage de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_site IS 'Remarques générale concernant la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_fct_majeur IS 'Remarques sur les fonctions majeurs de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_interet_patri IS 'Remarques sur les intérêts patrimoniaux de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_bilan_menace IS 'Remarques globales sur les menaces qui concernent la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_orient_act IS 'Remarques sur les orientations et les actes de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_usage_process IS 'Remarques concernant les usages et les processus naturels de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.code_cb IS 'Liste des codes CORINE Biotopes identifiés sur la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.lib_cb IS 'Liste des habitats CORINE Biotopes identifiés sur la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.activ_hum IS 'Liste des activités humaines identifiées sur la zone humide. Chaque élément de la liste est séparé par un '';'' et est lié à la colonne ''impacts'' et ''position''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.impact IS 'Liste des impacts identifiés et engendrés par les activités humaine sur la zone humide. Chaque élément de la liste est séparé par un '';'' et est lié à la colonne ''activ_hum'' et ''position''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides."position" IS 'Liste des potionnements des différentes activités humaines sur la zone humide. Chaque élément de la liste est séparé par un '';'' et est lié à la colonne ''activ_hum'' et ''impact''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_activ_hum IS 'Remarques concernant activités humaines identifiées sur la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.connexion IS 'Connexion de la zone humide dans son environnement.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.subm_orig IS 'Origine de l''eau submerssive.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.subm_freq IS 'Fréquence de la submersion de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.subm_etend IS 'Etendue de la submersion de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.fct_bio IS 'Liste des fonctions biologiques de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.fct_hydro IS 'Liste des fonctions hydrologiques de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.int_patri IS 'Liste des interêts patrimoniaux de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides."val_socioEco" IS 'Liste des valeurs socio-economiques de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.crit_delim IS 'Liste des critères de délimitation de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.crit_def_esp IS 'Liste des critères de l''espace de fonctionnalité de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.entree_eau_reg IS 'Liste des entrées d''eau du régime hydrique de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.entree_eau_perm IS 'Liste des permanances respectivement de chaque entrée d''eau de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.entree_eau_topo IS 'Liste de la toponymie respectivement de chaque entrée d''eau de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.sortie_eau_reg IS 'Liste des sorties d''eau du régime hydrique de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.sortie_eau_perm IS 'Liste des permanances respectivement de chaque sortie d''eau de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.sortie_eau_topo IS 'Liste de la toponymie respectivement de chaque sortie d''eau de la zone humide. Chaque élément de la liste est séparé par un '';''.';
"""
with con.begin() as cnx:
cnx.execute(text(v_zoneshumides))
cnx.execute(grant)
cnx.execute(comment)
v_rhomeosite = """
DROP VIEW IF EXISTS zones_humides.v_rhomeosite;
CREATE OR REPLACE VIEW zones_humides.v_rhomeosite
AS
SELECT
v.site_code||' - '||v.nom "NAME",
SPLIT_PART(v.auteur_geom,' (',1) "REFERENT",
REPLACE(SPLIT_PART(v.auteur_geom,' (',2),')','') "ORG",
SPLIT_PART(v.typo_sdage,' - ',1) "TYPE",
CASE WHEN r.nom::text = 'alpin' THEN '1'
WHEN r.nom::text = 'continental' THEN '2'
WHEN r.nom::text = 'mediterraneen' THEN '4'
END "ODONATE",
v.geom
FROM zones_humides.v_zoneshumides v, ref_territoire.ref_biogeo r
WHERE st_intersects(v.geom, r.geom)
"""
grant = """
GRANT ALL ON TABLE zones_humides.v_rhomeosite TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_rhomeosite TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(text(v_rhomeosite))
cnx.execute(grant)
test = """
SELECT * FROM zones_humides.v_zoneshumides;
"""
data = gpd.pd.read_sql_query(
sql = text(test),
con = con)
data[~data.index.isin(data.drop_duplicates(subset='site_code',keep=False).index)]
data[data.duplicated(subset='site_code',keep=False)]
3_AZALEE/create_view_zh3.py
+641
View File
@@ -0,0 +1,641 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-.
from sqlalchemy import text
from datetime import datetime as dt
import geopandas as gpd
from pycen import con
# Parametres bdd
from sqlalchemy import create_engine, text
from sqlalchemy.engine import URL
from pycen import con
drop_v_zh = 'DROP VIEW IF EXISTS zones_humides.v_zoneshumides CASCADE;'
with con.begin() as cnx:
cnx.execute(drop_v_zh)
v_zh_hab = """
DROP VIEW IF EXISTS zones_humides.v_zh_hab;
CREATE OR REPLACE VIEW zones_humides.v_zh_hab
AS
WITH author AS (
SELECT DISTINCT ON (id_sitehab)
c.id_sitehab,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM zones_humides.r_rsitehab_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_sitehab
ORDER BY 1
),
d_max as (
SELECT distinct on (id_site)
a.id_site,
c.auteur,
a."date"
FROM zones_humides.r_site_habitat a
JOIN author c ON c.id_sitehab = a.id
WHERE a."valid"
ORDER BY a.id_site, "date" DESC
)
SELECT --distinct on (a.id_cb)
d.id_site,
d.auteur,
d."date",
string_agg(a.id_cb,';' order by a.id_cb asc) code_cb,
string_agg(b.lb_hab_fr,';' order by a.id_cb asc) lib_cb
FROM (SELECT distinct on (id_site,id_cb) * FROM zones_humides.r_site_habitat ) a
JOIN ref_habitats.corine_biotope b ON a.id_cb = b.id
JOIN d_max d on a.id_site = d.id_site
WHERE a."valid"
GROUP BY 1,2,3
;"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zh_hab TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zh_hab TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(text(v_zh_hab))
cnx.execute(grant)
v_zh_usgprocess = """
DROP VIEW IF EXISTS zones_humides.v_zh_usgprocess;
CREATE OR REPLACE VIEW zones_humides.v_zh_usgprocess
AS
WITH auteur AS (
SELECT DISTINCT ON (id_siteusage)
c.id_siteusage,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM zones_humides.r_rsiteusage_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_siteusage
ORDER BY 1
), usgprocess as (
SELECT distinct on (a.id_site,a1.nom,a2.nom)
--a.id_geom_site,
a.id_site,
c.auteur,
a."date",
a1.nom activ_hum,
a.activ_hum_autre,
a2.nom impact,
a3.nom "position",
a.remarques rmq_activ_hum
FROM zones_humides.r_site_usageprocess a
JOIN zones_humides.param_activ_hum a1 ON a.id_activ_hum = a1.id
JOIN zones_humides.param_impact a2 ON a.id_impact = a2.id
JOIN zones_humides.param_position a3 ON a.id_position = a3.id
JOIN auteur c ON c.id_siteusage = a.id
WHERE a."valid"
--AND date = a.date
--GROUP BY a.id_geom_site,c.auteur,a1.nom,a.activ_hum_autre,a2.nom,a3.nom,a.remarques
--GROUP BY a.id_site,c.auteur,a1.nom,a.activ_hum_autre,a2.nom,a3.nom,a.remarques
ORDER BY a.id_site,a1.nom,a2.nom, a."date" desc,row_number() OVER (ORDER BY a.id_site) desc
)
SELECT DISTINCT ON (id_site)
--id_geom_site,
id_site,
auteur,
"date",
string_agg(
CASE WHEN activ_hum = 'Autre (préciser dans l''encart réservé aux remarques)' THEN activ_hum_autre
ELSE activ_hum END,';') activ_hum,
string_agg(
CASE WHEN impact = 'Autre (préciser dans l''encart réservé aux remarques)' THEN rmq_activ_hum
ELSE impact END,';') impact,
string_agg("position",';') "position",
string_agg(
CASE WHEN impact = 'Autre (préciser dans l''encart réservé aux remarques)' THEN NULL
ELSE rmq_activ_hum END,';') rmq_activ_hum
FROM usgprocess
GROUP BY 1,2,3
ORDER BY id_site, "date" desc,row_number() OVER (ORDER BY id_site) desc
;"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zh_usgprocess TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zh_usgprocess TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(v_zh_usgprocess)
cnx.execute(grant)
v_zh_connex = """
DROP VIEW IF EXISTS zones_humides.v_zh_connex;
CREATE OR REPLACE VIEW zones_humides.v_zh_connex
AS
WITH auteur AS (
SELECT DISTINCT ON (id_siteconnect)
c.id_siteconnect,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM zones_humides.r_rsiteconnect_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_siteconnect
ORDER BY 1
)
SELECT DISTINCT ON (id_site)
--a.id_geom_site,
a.id_site,
c.auteur,
a."date",
b.nom connexion
FROM zones_humides.r_site_type_connect a
JOIN zones_humides.param_type_connect b ON a.id_param_connect = b.id
JOIN auteur c ON c.id_siteconnect = a.id
WHERE a."valid"
--AND a.date = (SELECT MAX(date) FROM zones_humides.r_site_type_connect WHERE id_site = a.id_site)
ORDER BY a.id_site,"date" desc,row_number() OVER (ORDER BY a.id_site) desc
;"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zh_connex TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zh_connex TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(v_zh_connex)
cnx.execute(grant)
v_zh_submertion = """
DROP VIEW IF EXISTS zones_humides.v_zh_submertion;
CREATE OR REPLACE VIEW zones_humides.v_zh_submertion
AS
WITH auteur AS (
SELECT DISTINCT ON (id_sitesub)
c.id_sitesub,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM zones_humides.r_rsitesub_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_sitesub
ORDER BY 1
)
SELECT DISTINCT ON (a.id_site)
--a.id_geom_site,
a.id_site,
c.auteur,
a."date",
a1.nom subm_freq,
a2.nom subm_etend,
a.id_origsub subm_orig
FROM zones_humides.r_site_sub a
JOIN (zones_humides.param_sub a1 JOIN zones_humides.type_param_sub s6411 ON s6411.id = a1.id_type) ON a.id_freqsub = a1.id
JOIN (zones_humides.param_sub a2 JOIN zones_humides.type_param_sub s6421 ON s6421.id = a2.id_type) ON a.id_etendsub = a2.id
JOIN auteur c ON c.id_sitesub = a.id
WHERE a."valid"
--AND a.date = (SELECT MAX(date) FROM zones_humides.r_site_sub WHERE id_site = a.id_site)
AND s6411.nom = 'Submersion fréquente'
AND s6421.nom = 'Submersion étendue'
ORDER BY a.id_site,"date" desc,row_number() OVER (ORDER BY a.id_site) desc
;"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zh_submertion TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zh_submertion TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(v_zh_submertion)
cnx.execute(grant)
v_zh_fctecosociopatri_cross = """
DROP VIEW IF EXISTS zones_humides.v_zh_fctecosociopatri_cross;
CREATE OR REPLACE VIEW zones_humides.v_zh_fctecosociopatri_cross
AS
WITH auteur AS (
SELECT DISTINCT ON (id_sitefct)
c.id_sitefct,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM zones_humides.r_rsitefct_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_sitefct
ORDER BY 1
),max_author as (
SELECT DISTINCT ON (id_site)
a.id_site,
--(SELECT regexp_split_to_table(auth,',') FROM (values (string_agg(auteur,',' order by ROW()))) AS value(auth) Limit 1) auteur,
auteur,
a."date"
FROM auteur
JOIN zones_humides.r_site_fctecosociopatri a ON id_sitefct = a.id
order by 1,3 DESC,2
)
SELECT DISTINCT ON (id_site)
--id_geom_site,
id_site,
--(SELECT auteur ORDER BY ct."date" DESC) "date",
"date",
auteur,
fct_bio,
fct_hydro,
int_patri,
"val_socioEco"
FROM crosstab(
$$ WITH tmp_select as (
SELECT
max(a."id") id_sitefct,
id_site,
--(SELECT regexp_split_to_table(auth,',') FROM (values (string_agg(d.auteur,',' order by ROW()))) AS value(auth) Limit 1) auteur,
--MAX(a.date) date,
c.nom_court type_param,
b.nom,
string_agg(DISTINCT a.description,'\n') remark
FROM zones_humides.r_site_fctecosociopatri a
JOIN (zones_humides.param_fct_eco_socio_patri b
JOIN zones_humides.type_param_fct c ON b.id_type = c.id
) ON a.id_fct = b.id
--JOIN auteur d ON d.id_sitefct = a.id
WHERE a."valid"
GROUP BY a.id_site,c.nom_court,b.nom
ORDER BY a.id_site,c.nom_court DESC,b.nom
)
SELECT
id_site,
--(SELECT regexp_split_to_table(auth,',') FROM (values (string_agg(d.auteur,',' order by ROW()))) AS value(auth) Limit 1) auteur,
--MAX("date") "date",
type_param,
string_agg(distinct
CASE WHEN remark IS NULL THEN nom
WHEN remark = '' THEN nom
ELSE CONCAT(nom,' (',remark,')') END,
';') fct
FROM tmp_select
--JOIN auteur d using (id_sitefct)
GROUP BY id_site,type_param
ORDER BY id_site,type_param;$$,
$$SELECT DISTINCT nom_court FROM zones_humides.type_param_fct ORDER BY 1 ASC;$$
) AS ct (
"id_site" text,
--"auteur" text,
--"date" date,
"fct_bio" text,
"fct_hydro" text,
"int_patri" text,
"val_socioEco" text
)
JOIN max_author USING (id_site)
--GROUP BY id_site,auteur,fct_bio,fct_hydro,int_patri,"val_socioEco"
--WHERE id_site in ('38CG0110','38RD0057','38VS0063')
ORDER BY id_site,row_number() OVER (ORDER BY id_site) desc
;"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zh_fctecosociopatri_cross TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zh_fctecosociopatri_cross TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(v_zh_fctecosociopatri_cross)
cnx.execute(grant)
v_zh_critdelim_cross = """
DROP VIEW IF EXISTS zones_humides.v_zh_critdelim_cross;
CREATE OR REPLACE VIEW zones_humides.v_zh_critdelim_cross
AS
WITH auteur AS (
SELECT DISTINCT ON (id_sitedelim)
c.id_sitedelim,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM zones_humides.r_rsitedelim_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_sitedelim
ORDER BY 1
),max_author as (
SELECT DISTINCT ON (id_site)
a.id_site,
--(SELECT regexp_split_to_table(auth,',') FROM (values (string_agg(auteur,',' order by ROW()))) AS value(auth) Limit 1) auteur,
auteur,
a."date"
FROM auteur
JOIN zones_humides.r_site_critdelim a ON id_sitedelim = a.id
order by 1,3 DESC,2
)
SELECT DISTINCT ON (id_site)
--id_geom_site,
id_site,
"date",
auteur,
crit_delim,
crit_def_esp
FROM crosstab(
$$ SELECT
--a.id_geom_site::bigint,
id_site,
c.nom_court type_param,
string_agg( DISTINCT
CASE WHEN a.description IS NULL THEN b.nom
WHEN a.description = '' THEN b.nom
ELSE CONCAT(b.nom,' (',a.description,')') END,
'; ') fct
FROM zones_humides.r_site_critdelim a
JOIN (zones_humides.param_delim_fct b
JOIN zones_humides.type_param_delim_fct c ON b.id_type = c.id
) ON a.id_crit_delim = b.id
--JOIN auteur d ON d.id_sitedelim = a.id
WHERE a."valid"
GROUP BY 1,2
ORDER BY 1$$,
$$SELECT DISTINCT nom_court FROM zones_humides.type_param_delim_fct ORDER BY 1 DESC;$$
) AS ct (
"id_site" text,
--"auteur" text,
--"date" date,
"crit_delim" text,
"crit_def_esp" text)
JOIN max_author USING (id_site)
--GROUP BY id_site,auteur,crit_delim,crit_def_esp
--WHERE id_site in ('38CG0110','38RD0057','38VS0063')
ORDER BY id_site,row_number() OVER (ORDER BY id_site) desc
;"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zh_critdelim_cross TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zh_critdelim_cross TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(v_zh_critdelim_cross)
cnx.execute(grant)
v_zh_reghydro_cross = """
DROP VIEW IF EXISTS zones_humides.v_zh_reghydro_cross;
CREATE OR REPLACE VIEW zones_humides.v_zh_reghydro_cross
AS
SELECT DISTINCT ON (id_site)
--id_geom_site,
id_site,
auteur,
"date",
(string_to_array(ct.entree_eau,'//'))[1] entree_eau_reg,
(string_to_array(ct.entree_eau,'//'))[2] entree_eau_perm,
(string_to_array(ct.entree_eau,'//'))[3] entree_eau_topo,
(string_to_array(ct.sortie_eau,'//'))[1] sortie_eau_reg,
(string_to_array(ct.sortie_eau,'//'))[2] sortie_eau_perm,
(string_to_array(ct.sortie_eau,'//'))[3] sortie_eau_topo
FROM crosstab(
$$WITH auteur AS (
SELECT DISTINCT ON (id_sitehydro)
c.id_sitehydro,
string_agg(c1.auteur,';' ORDER BY c1.auteur) auteur
FROM zones_humides.r_rsitehydro_auteur c
JOIN personnes.v_personne c1 ON c1.id = c.id_auteur
GROUP BY c.id_sitehydro
ORDER BY 1
)
SELECT
--a.id_geom_site,
id_site,
d.auteur,
MAX(a.date) date,
a.in_out,
CONCAT(
string_agg(a1.nom,';'),'//', -- reg_hydro
string_agg(a2.nom,';'),'//', -- permanence
string_agg(a.rmq_toponymie,';') -- rmq_toponymie
) hydro
FROM zones_humides.r_site_reghydro a
LEFT JOIN zones_humides.param_reg_hydro a1 ON a.id_reg_hydro = a1.id
LEFT JOIN zones_humides.param_permanence a2 ON a.id_permanence = a2.id
JOIN auteur d ON d.id_sitehydro = a.id
WHERE a."valid"
GROUP BY a.id_site,d.auteur,a.in_out
ORDER BY 1,2$$,
$$SELECT DISTINCT in_out FROM zones_humides.r_site_reghydro ORDER BY 1 DESC;$$
) AS ct (
"id_site" text,
"auteur" text,
"date" date,
"entree_eau" text,
"sortie_eau" text)
--GROUP BY id_site,auteur,entree_eau,sortie_eau
ORDER BY id_site,"date" desc,row_number() OVER (ORDER BY id_site) desc
;"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zh_reghydro_cross TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zh_reghydro_cross TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(v_zh_reghydro_cross)
cnx.execute(grant)
v_zoneshumides = """
DROP VIEW IF EXISTS zones_humides.v_zoneshumides CASCADE;
CREATE OR REPLACE VIEW zones_humides.v_zoneshumides
AS
WITH tmp_auteur_site as (
SELECT
s0.id,
string_agg(s11.auteur,';' ORDER BY s11.auteur) AS auteur_site
FROM sites.sites s0
JOIN sites.type_milieu s3 ON s0.id_type_milieu = s3.id
LEFT JOIN (sites.r_sites_auteur s1
JOIN personnes.v_personne s11 ON s1.id_auteur = s11.id
) ON s0.id::text = s1.id_site::text
WHERE s0.date_fin IS NULL
AND s3.nom_court::text = 'Milieux humides'::text
GROUP BY s0.id
ORDER BY s0.id
), tmp_auteur_geom as (
SELECT
s6.id,
string_agg(s611.auteur,';' ORDER BY s611.auteur) AS auteur_geom
FROM sites.sites s0
JOIN sites.type_milieu s3 ON s0.id_type_milieu = s3.id
LEFT JOIN (sites.r_sites_geom s6
LEFT JOIN (sites.r_geomsites_auteur s61
JOIN personnes.v_personne s611 ON s61.id_auteur = s611.id
) ON s6.id = s61.id_geom_site
) ON s0.id = s6.id_site
WHERE s0.date_fin IS NULL
AND s3.nom_court::text = 'Milieux humides'::text
GROUP BY s6.id
ORDER BY s6.id
),
tmp_other_name as (
SELECT
id_site_new,
ARRAY_AGG(id_site_old) id_site_old
FROM sites.r_site_maj
WHERE id_site_old IS NOT NULL
GROUP BY id_site_new
)
SELECT
s0.id AS site_code,
s7.id_site_old old_code,
s0.nom,
s2.autre_nom,
s1.auteur_site,
s6.auteur_geom,
(
SELECT DISTINCT ON (date) auteur
FROM (values (s6.date,s6.auteur_geom),(s62.date,s62.auteur),(s63.date,s63.auteur),(s64.date,s64.auteur),(s65.date,s65.auteur),
(s66.date,s66.auteur),(s67.date,s67.auteur),(s68.date,s68.auteur)) AS value(date,auteur)
WHERE date = (SELECT MAX(date) FROM (values (s6.date),(s62.date),(s63.date),(s64.date),(s65.date),(s66.date),(s67.date),(s68.date)) AS value(date))
) as auteur_last_maj,
s0.date_deb AS date_site,
s6.date AS date_geom,
(SELECT MAX(date) FROM (values (s6.date),(s62.date),(s63.date),(s64.date),(s65.date),(s66.date),(s67.date),(s68.date)) AS value(date)) as date_last_maj,
s3.nom_court AS type_milieu,
s4.nom AS type_site,
s5.id||' - '||s5.nom AS typo_sdage,
s5.mnemo mnemo_sdage,
s0.remarques AS rmq_site,
s6.rmq_fct_majeur,
s6.rmq_interet_patri,
s6.rmq_bilan_menace,
s6.rmq_orient_act,
s6.rmq_usage_process,
s62.code_cb,
s62.lib_cb,
s63.activ_hum,
s63.impact,
s63."position",
s63.rmq_activ_hum,
s64.connexion,
s65.subm_orig,
s65.subm_freq,
s65.subm_etend,
s66.fct_bio,
s66.fct_hydro,
s66.int_patri,
s66."val_socioEco",
s67.crit_delim,
s67.crit_def_esp,
s68.entree_eau_reg,
s68.entree_eau_perm,
s68.entree_eau_topo,
s68.sortie_eau_reg,
s68.sortie_eau_perm,
s68.sortie_eau_topo,
s6.id_origine,
s6.geom
FROM sites.sites s0
LEFT JOIN tmp_auteur_site s1 ON s0.id = s1.id
LEFT JOIN sites.autre_nom s2 ON s0.id = s2.id_site
LEFT JOIN sites.type_milieu s3 ON s0.id_type_milieu = s3.id
LEFT JOIN sites.type_site s4 ON s0.id_type_site::text = s4.id::text
LEFT JOIN sites.typo_sdage s5 ON s0.id_typo_sdage::text = s5.id::text
LEFT JOIN (SELECT DISTINCT ON (sg.id_site) *
FROM sites.r_sites_geom sg
LEFT JOIN tmp_auteur_geom s61 USING (id) --ON s6.id = s61.id
ORDER BY sg.id_site, "date" DESC, date_insert DESC
) s6 ON s0.id::text = s6.id_site::text
LEFT JOIN zones_humides.v_zh_hab s62 ON s0.id = s62.id_site
LEFT JOIN zones_humides.v_zh_usgprocess s63 ON s0.id = s63.id_site
LEFT JOIN zones_humides.v_zh_connex s64 ON s0.id = s64.id_site
LEFT JOIN zones_humides.v_zh_submertion s65 ON s0.id = s65.id_site
LEFT JOIN zones_humides.v_zh_fctecosociopatri_cross s66 ON s0.id = s66.id_site
LEFT JOIN zones_humides.v_zh_critdelim_cross s67 ON s0.id = s67.id_site
LEFT JOIN zones_humides.v_zh_reghydro_cross s68 ON s0.id = s68.id_site
LEFT JOIN tmp_other_name s7 ON s0.id = s7.id_site_new
WHERE s0.date_fin IS NULL
AND s3.nom_court::text = 'Milieux humides'::text
AND s6.date = (SELECT MAX(date) FROM sites.r_sites_geom WHERE r_sites_geom.id_site = s0.id)
ORDER BY s0.id ASC NULLS FIRST;
"""
grant = """
GRANT ALL ON TABLE zones_humides.v_zoneshumides TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_zoneshumides TO grp_consult;
"""
comment = """
COMMENT ON COLUMN zones_humides.v_zoneshumides.site_code IS 'Identifiant de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.old_code IS 'Ancien identifiant de la zone humide pouvant se retrouver dans des données historiques.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.nom IS 'Nom de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.autre_nom IS 'Autre nom possible de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.auteur_site IS 'Auteur ayant caractérisé la zone humide pour la première fois.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.auteur_geom IS 'Auteur ayant définis la géometrie actuelle de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.auteur_last_maj IS 'Auteur le plus récent ayant défini les attributs de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.date_site IS 'Date de la première description de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.date_geom IS 'Date de la géométrie.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.date_last_maj IS 'Date des attributs les plus récents.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.type_milieu IS 'Caractérisation du milieu.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.type_site IS '';
COMMENT ON COLUMN zones_humides.v_zoneshumides.typo_sdage IS 'Typologie sdage de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.mnemo_sdage IS 'Libellé mnémonique sdage de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_site IS 'Remarques générale concernant la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_fct_majeur IS 'Remarques sur les fonctions majeurs de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_interet_patri IS 'Remarques sur les intérêts patrimoniaux de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_bilan_menace IS 'Remarques globales sur les menaces qui concernent la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_orient_act IS 'Remarques sur les orientations et les actes de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_usage_process IS 'Remarques concernant les usages et les processus naturels de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.code_cb IS 'Liste des codes CORINE Biotopes identifiés sur la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.lib_cb IS 'Liste des habitats CORINE Biotopes identifiés sur la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.activ_hum IS 'Liste des activités humaines identifiées sur la zone humide. Chaque élément de la liste est séparé par un '';'' et est lié à la colonne ''impacts'' et ''position''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.impact IS 'Liste des impacts identifiés et engendrés par les activités humaine sur la zone humide. Chaque élément de la liste est séparé par un '';'' et est lié à la colonne ''activ_hum'' et ''position''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides."position" IS 'Liste des potionnements des différentes activités humaines sur la zone humide. Chaque élément de la liste est séparé par un '';'' et est lié à la colonne ''activ_hum'' et ''impact''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_activ_hum IS 'Remarques concernant activités humaines identifiées sur la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.connexion IS 'Connexion de la zone humide dans son environnement.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.subm_orig IS 'Origine de l''eau submerssive.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.subm_freq IS 'Fréquence de la submersion de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.subm_etend IS 'Etendue de la submersion de la zone humide.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.fct_bio IS 'Liste des fonctions biologiques de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.fct_hydro IS 'Liste des fonctions hydrologiques de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.int_patri IS 'Liste des interêts patrimoniaux de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides."val_socioEco" IS 'Liste des valeurs socio-economiques de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.crit_delim IS 'Liste des critères de délimitation de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.crit_def_esp IS 'Liste des critères de l''espace de fonctionnalité de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.entree_eau_reg IS 'Liste des entrées d''eau du régime hydrique de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.entree_eau_perm IS 'Liste des permanances respectivement de chaque entrée d''eau de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.entree_eau_topo IS 'Liste de la toponymie respectivement de chaque entrée d''eau de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.sortie_eau_reg IS 'Liste des sorties d''eau du régime hydrique de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.sortie_eau_perm IS 'Liste des permanances respectivement de chaque sortie d''eau de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.sortie_eau_topo IS 'Liste de la toponymie respectivement de chaque sortie d''eau de la zone humide. Chaque élément de la liste est séparé par un '';''.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.id_origine IS 'Identifiant d''origine de la zone fournie par un partenaire exterieur.';
COMMENT ON COLUMN zones_humides.v_zoneshumides.geom IS 'Géometrie de la zone humide.';
"""
with con.begin() as cnx:
cnx.execute(text(v_zoneshumides))
cnx.execute(grant)
cnx.execute(comment)
v_rhomeosite = """
DROP VIEW IF EXISTS zones_humides.v_rhomeosite;
CREATE OR REPLACE VIEW zones_humides.v_rhomeosite
AS
with t1 as (
SELECT
v.site_code,
r.nom,
st_area(st_intersection(v.geom, r.geom)) area_intersect
FROM zones_humides.v_zoneshumides v
join ref_territoire.ref_biogeo r on st_intersects(v.geom, r.geom)
--group by 1,2
), t2 as (
select
site_code,
nom,
row_number() over (partition by site_code order by area_intersect desc) as ismax_ter
from t1
)
SELECT (v.site_code::text || ' - '::text) || v.nom::text AS "NAME",
split_part(v.auteur_geom, ' ('::text, 1) AS "REFERENT",
replace(split_part(v.auteur_geom, ' ('::text, 2), ')'::text, ''::text) AS "ORG",
split_part(v.typo_sdage, ' - '::text, 1) AS "TYPE",
CASE
WHEN t2.nom = 'alpin'::text THEN '1'::text
WHEN t2.nom = 'continental'::text THEN '2'::text
WHEN t2.nom = 'mediterraneen'::text THEN '4'::text
ELSE NULL::text
END AS "ODONATE",
v.geom
FROM zones_humides.v_zoneshumides v
join t2 on v.site_code = t2.site_code and t2.ismax_ter = 1
;
"""
grant = """
GRANT ALL ON TABLE zones_humides.v_rhomeosite TO grp_admin;
GRANT SELECT ON TABLE zones_humides.v_rhomeosite TO grp_consult;
"""
with con.begin() as cnx:
cnx.execute(text(v_rhomeosite))
cnx.execute(grant)
test = """
SELECT * FROM zones_humides.v_zoneshumides;
"""
data = gpd.pd.read_sql_query(
sql = text(test),
con = con)
data[~data.index.isin(data.drop_duplicates(subset='site_code',keep=False).index)]
data[data.duplicated(subset='site_code',keep=False)]
3_AZALEE/del_psduplicated.py
+187
View File
@@ -0,0 +1,187 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Nom : : del_psduplicated.py
#Description : Suppessio, des géométries en doubon sous un nom différents et avec des erreurs d'intégration des données
#Copyright : 2022-07-20, CEN38
#Auteur : Colas Geier
#Version : 1.0
import geopandas as gpd
import pycen
from pycen import con,update_to_sql
df1 = gpd.read_postgis('SELECT * FROM ps.v_covered_ps ORDER BY 2',con,geom_col='geom_old')
df2 = gpd.read_postgis('SELECT * FROM ps.v_covered_ps ORDER BY 2',con,geom_col='geom_new')
# df.reset_index(inplace=True, drop=False)
df1.to_postgis('covered_ps_old',con,'ps','replace',False,geom_col='geom_old')
df2.to_postgis('covered_ps_new',con,'ps','replace',False,geom_col='geom_new')
df = df1.copy()
# ('38BDAU0907','38BIEV0032','38BIEV0033','38BDAU0905','38PCHA1088','38PCHA1089','38PCHA1002','38PCHA0908','38VERC3112','38PCHA0316','38VERC3171','38PCHA0325','38VERC2662','38PCHA0232','38PCHA0244','38VERC2732','38PCHA1086','38PCHA1087','38VERC3206','38PCHA0332','38PCHA0297','38VERC3013','38VERC2983','38PCHA0291','38VERC2961','38PCHA0286','38VERC3212','38PCHA0333','38PCHA0326','38VERC3175','38PCHA1005','38PCHA0018','38PCHA0474','38VERC3681','38PCHA0044','38PCHA1007','38PCHA0910','38PCHA1003','38PCHA0274','38VERC2902','38PCHA0032','38PCHA1006','38PCHA0121','38VERC1824','38VERC1743','38PCHA0111','38PCHA1004','38PCHA0928','38VERC1848','38PCHA0123','38PCHA0057','38PCHA1008','38PCHA0065','38PCHA1009','38BDAU0908','38VERC2008','38VERC2495','38BELL0154','38GRES0637','38VERC3451','38VERC0383','38VERC3222','38VERC3348','38GRES0629','38VERC2912','38CHAR0502','38CHAR0045','38VERC2058','38VERC1931','38MATH0010','38VERC1924','38CHAR0040','38GRES0006','38CHAR0900','38BELL0030','38VERC2163','38GRES0066','38VERC2100','38CHAR0049','38VERC2187','38GRES0077','38VERC2272','38VERC2807','38CHAR0079','38VERC2658','38CHAR0066','38CHAR0100','38VERC3008','38CHAR0903','38BELL0005','38CHAR0901','38CHAR0008','38VERC2414','38VERC0219','38VERC2514','38GRES0084','38VERC3076','38GRES0119','38VERC3311','38GRES0132','38VERC2718','38CHAR0072','38VERC3205','38CHAR0115','38VERC3108','38BELL0049','38VERC2377','38VERC0215','38VERC2387','38VERC0216','38VERC2431','38BELL0035','38VERC2408','38GRES0080','38GRES0686','38VERC3623','38CHAR0468','38VERC2384','38MATH0111','38VERC2649','38BELL0198','38VERC2661','38GRES0437','38VERC1748','38VERC2614','38GRES0472','38GRES0462','38VERC2446','38TRIE0215','38VERC2356','38VERC2329','38TRIE0211','38VERC2315','38TRIE0210','38TRIE0222','38VERC2405','38VERC2439','38TRIE0227','38TRIE0228','38VERC2443','38TRIE0229','38VERC2455','38CHAR0902','38TRIE0022','38GRES0141','38VERC3408','38VERC3159','38CHAR0113','38CHAR0001','38CHAR0899','38CHAR0905','38GRES0028','38CHAR0906','38CHAR0012','38CHAR0017','38CHAR0908','38GRES0136','38VERC3376','38VERC1668','38GRES0053','38GRES0127','38VERC3197','38VERC1549','38GRES0045','38GRES0030','38CHAR0907','38GRES1039','38GRES0031','38BIEV0004','38VERC3399','38CHAR0904','38GRES0025','38CHAR0029','38VERC1533','38CHAR0133','38VERC3354','38CHAR0031','38VERC1618','38VERC3369','38CHAR0135','38VERC3464','38GRES0143','38GRES0016','38GRES1038','38BELL0192','38VERC2556','38VERC2513','38CHAR0402','38BELL0164','38BELL0492','38MATH0024','38VERC2487','38VERC2646','38TRIE0250','38VERC3523','38TRIE0377','38BELL0489','38BELL0155','38BELL0491','38MCHA0009','38BELL0490','38BELL0162','38VERC1687','38CHAR0034','38VERC1721','38BELL0025','38BELL0016','38CHAR0909','38MATH0005','38VERC1756','38BELL0043','38VERC2641','38CHAR0027','38CHAR0910','38GRES0089','38VERC2601','38GRES0092','38VERC2629')
# ('38BIEV0032','38BDAU0907','38BDAU0905','38BIEV0033','38PCHA1089','38PCHA1088','38PCHA1002','38PCHA0908','38VERC3112','38PCHA0316','38VERC3171','38PCHA0325','38VERC2662','38PCHA0232','38VERC2732','38PCHA0244','38PCHA1086','38PCHA1087','38VERC3206','38PCHA0332','38VERC3013','38PCHA0297','38PCHA0291','38VERC2983','38VERC2961','38PCHA0286','38PCHA0333','38VERC3212','38PCHA0326','38VERC3175','38PCHA1005','38PCHA0018','38VERC3681','38PCHA0474','38PCHA1007','38PCHA0044','38PCHA0910','38PCHA1003','38VERC2902','38PCHA0274','38PCHA1006','38PCHA0032','38VERC1824','38PCHA0121','38PCHA0111','38VERC1743','38PCHA0928','38PCHA1004','38VERC1848','38PCHA0123','38PCHA0057','38PCHA1008','38PCHA1009','38PCHA0065','38BDAU0908','38VERC2008','38VERC2495','38BELL0154','38GRES0637','38VERC3451','38VERC0383','38VERC3222','38GRES0629','38VERC3348','38CHAR0502','38VERC2912','38VERC3623','38GRES0686','38CHAR0468','38VERC2384','38TRIE0215','38VERC2356','38VERC2329','38TRIE0211','38VERC2315','38TRIE0210','38VERC2405','38TRIE0222','38VERC2439','38TRIE0227','38VERC2443','38TRIE0228','38VERC2455','38TRIE0229','38VERC2487','38MATH0024','38VERC2646','38TRIE0250','38VERC3523','38TRIE0377')
# ('38BDAU0907','38BIEV0032','38BDAU0905','38BIEV0033','38PCHA1089','38PCHA1088','38PCHA1002','38PCHA0908','38PCHA0316','38VERC3112','38VERC3171','38PCHA0325','38PCHA0232','38VERC2662','38VERC2732','38PCHA0244','38PCHA1086','38PCHA1087','38VERC3206','38PCHA0332','38VERC3013','38PCHA0297','38PCHA0291','38VERC2983','38PCHA0286','38VERC2961','38PCHA0333','38VERC3212','38PCHA0326','38VERC3175','38PCHA0018','38PCHA1005','38PCHA0474','38VERC3681','38PCHA0044','38PCHA1007','38PCHA0910','38PCHA1003','38VERC2902','38PCHA0274','38PCHA0032','38PCHA1006','38VERC1824','38PCHA0121','38VERC1743','38PCHA0111','38PCHA0928','38PCHA1004','38PCHA0123','38VERC1848','38PCHA0057','38PCHA1008','38PCHA0065','38PCHA1009')
# ('38BDAU0907','38BIEV0032','38BDAU0905','38BIEV0033','38PCHA1088','38PCHA1089','38PCHA0316','38VERC3112','38PCHA0325','38VERC3171','38PCHA0232','38VERC2662','38PCHA0244','38VERC2732','38PCHA1087','38PCHA1086','38VERC3206','38PCHA0332','38VERC3013','38PCHA0297','38PCHA0291','38VERC2983','38PCHA0286','38VERC2961','38PCHA0333','38VERC3212','38VERC3175','38PCHA0326','38PCHA0018','38PCHA1005','38VERC3681','38PCHA0474','38PCHA0044','38PCHA1007','38PCHA0274','38VERC2902','38PCHA1006','38PCHA0032','38VERC1824','38PCHA0121','38VERC1743','38PCHA0111','38VERC1848','38PCHA0123','38PCHA0057','38PCHA1008','38PCHA0065','38PCHA1009')
# ('38BDAU0907','38BIEV0032','38BDAU0905','38BIEV0033','38PCHA1089','38PCHA1088','38PCHA0316','38VERC3112','38PCHA0325','38VERC3171','38PCHA0232','38VERC2662','38VERC2732','38PCHA0244','38PCHA1086','38PCHA1087','38VERC3206','38PCHA0332','38VERC3013','38PCHA0297','38VERC2983','38PCHA0291','38VERC2961','38PCHA0286','38PCHA0333','38VERC3212','38VERC3175','38PCHA0326','38PCHA0018','38PCHA1005','38PCHA0274','38VERC2902','38PCHA0121','38VERC1824','38PCHA0111','38VERC1743','38VERC1848','38PCHA0123','38PCHA1008','38PCHA0057','38PCHA1009','38PCHA0065')
# ('38BIEV0032','38BDAU0907','38BIEV0033','38BDAU0905','38PCHA1088','38PCHA1089','38PCHA1087','38PCHA1086')
v_ps = gpd.read_postgis(
"SELECT * FROM ps.v_pelouseseches WHERE site_code in ('%s') ORDER BY source,geom"%"','".join(df.id_site.tolist())
,pycen.con)
v_ps.set_index(['site_code','district_nat'],inplace=True)
v_ps.dropna(axis=1,how='all', inplace=True)
v_ps.fillna('NA', inplace=True)
for col in v_ps.columns[v_ps.columns.str.startswith('auteur')]:
tmp = v_ps[col].str.split(';')
for x in tmp: x.sort()
v_ps[col] = [';'.join(x) for x in tmp]
vp_clean = v_ps.drop_duplicates()
v_ps.loc[~v_ps.index.isin(vp_clean.index)]
v_ps.loc[~v_ps.index.isin(vp_clean.index)].droplevel(-1).index.tolist()
# ('38CHAR0915','38CHAR0919','38CHAR0922','38CHAR0936','38CHAR0938','38CHAR0940','38CHAR1195','38CHAR1197','38CHAR1203','38CHAR1220','38CHAR1221','38CHAR1222','38CHAR1223','38CHAR1224','38CHAR1225','38CHAR1226','38CHAR1227','38CHAR1228','38CHAR1229','38CHAR1230','38CHAR1231','38CHAR1232','38CHAR1233','38CHAR1234','38CHAR1235','38CHAR1236','38CHAR1237','38CHAR1238','38CHAR1239','38CHAR1240','38CHAR1241','38CHAR1242','38CHAR1243','38CHAR1244','38CHAR1245','38GRES1073','38GRES1074','38GRES1075','38MCHA0049','38PCHA1083','38PCHA1208','38PCHA1210','38PCHA1215','38PCHA1221','38TRIE1173','38TRIE2416','38TRIE2481','38TRIE2548','38TRIE2563','38TRIE2590','38TRIE2592','38TRIE2594','38TRIE2595','38TRIE2596','38TRIE2597','38TRIE2598','38TRIE2599','38TRIE2602','38TRIE2604','38TRIE2605','38TRIE2607','38TRIE2608','38TRIE2610','38TRIE2611','38TRIE2614','38TRIE2615','38TRIE2616','38TRIE2618','38TRIE2619','38TRIE2620','38TRIE2621','38TRIE2622','38TRIE2623','38TRIE2624','38TRIE2625','38TRIE2626','38TRIE2627','38TRIE2628','38TRIE2629','38TRIE2631','38TRIE2632','38TRIE2633','38TRIE2634','38TRIE2635','38TRIE2636','38TRIE2637','38TRIE2638','38TRIE2639','38TRIE2640','38TRIE2641','38TRIE2644','38TRIE2645','38TRIE2646','38TRIE2647','38TRIE2648','38TRIE2650','38TRIE2651','38TRIE2652','38TRIE2653','38TRIE2654','38TRIE2656','38TRIE2657','38TRIE2658','38TRIE2659','38TRIE2662','38TRIE2663','38TRIE2664','38TRIE2665','38TRIE2666','38TRIE2667','38TRIE2668','38TRIE2669','38TRIE2670','38TRIE2671','38TRIE2672','38TRIE2674','38TRIE2675','38TRIE2676','38TRIE2677','38TRIE2678','38TRIE2679','38TRIE2680','38TRIE2682','38TRIE2683','38TRIE2688','38TRIE2689','38TRIE2690','38TRIE2693','38TRIE2695','38TRIE2696','38TRIE2697','38TRIE2698','38TRIE2699','38TRIE2700','38TRIE2701','38TRIE2702','38TRIE2703','38TRIE2704','38TRIE2705','38TRIE2707','38TRIE2708','38TRIE2710','38TRIE2712','38TRIE2713','38TRIE2714','38TRIE2715','38TRIE2720','38TRIE2721','38TRIE2722','38TRIE2723','38TRIE2724','38TRIE2727','38TRIE2731','38TRIE2735','38TRIE2736','38TRIE2739','38TRIE2741','38TRIE2743','38TRIE2744','38TRIE2749','38TRIE2750','38TRIE2758','38TRIE2760','38TRIE2762','38TRIE2764','38TRIE2765','38TRIE2766','38TRIE2767','38TRIE2768','38TRIE2769','38TRIE2771','38TRIE2776','38TRIE2780','38TRIE2788','38VERC3691','38VERC3692','38VERC3698','38VERC3701','38VERC3903','38VERC3904','38VERC3906','38VERC3921','38VERC3924','38VERC3936','38VERC3937','38VERC3940','38VERC3941','38VERC3943','38VERC3945','38VERC3946','38VERC3948','38VERC3949','38VERC3950','38VERC3953','38VERC4017','38VERC4104','38VERC4123','38VERC4124','38VERC4180','38VERC4187','38VERC4188','38VERC4197','38VERC4205','38VERC4218','38VERC4290','38VERC4294','38VERC4364','38VERC4392','38VERC4396','38VERC4405','38VERC4406','38VERC4407','38VERC4408','38VERC4409','38VERC4446','38VERC4447','38VERC4448','38VERC4453','38VERC4462','38VERC4464','38VERC4469','38VERC4484','38VERC4485','38VERC4492','38VERC4498','38VERC4512','38VERC4526','38VERC4527','38VERC4530','38VERC4544','38VERC4558','38VERC4576','38VERC4577','38VERC4578','38VERC4579','38VERC4585','38VERC4588','38VERC4590','38VERC4627','38VERC4628','38VERC4636','38VERC4638','38VERC4639','38VERC4640','38VERC4643','38VERC4649','38VERC4650','38VERC4654','38VERC4655','38VERC4656','38VERC4657','38VERC4658','38VERC4659','38VERC4660','38VERC4661','38VERC4662','38VERC4663','38VERC4664','38VERC4665','38VERC4666','38VERC4667','38VERC4668','38VERC4669','38VERC4670','38VERC4671','38VERC4672','38VERC4673','38VERC4674','38VERC4675','38VERC4676','38VERC4678','38VERC4679','38VERC4680','38VERC4681','38VERC4685','38VERC4686','38VERC4692')
'38GRES1029','38GRES1031','38VERC0811','38VERC0861','38VERC1045','38VERC1427','38VERC1433','38VERC1453','38VERC1456','38VERC1465','38VERC1489','38VERC1494','38VERC3704','38VERC3708','38VERC4129','38VERC4134'
'38BIEV0032','38BDAU0907','38BDAU0905','38BIEV0033','38PCHA1089','38PCHA1088','38PCHA1002','38PCHA0908','38VERC3112','38PCHA0316','38PCHA0906','38PCHA1001','38PCHA0325','38VERC3171','38PCHA0232','38VERC2662','38VERC2732','38PCHA0244','38PCHA1086','38PCHA1087','38PCHA0332','38VERC3206','38VERC3013','38PCHA0297','38VERC2983','38PCHA0291','38VERC2961','38PCHA0286','38PCHA0333','38VERC3212','38VERC3175','38PCHA0326','38PCHA0018','38PCHA1005','38PCHA0474','38VERC3681','38PCHA0044','38PCHA1007','38PCHA0910','38PCHA1003','38PCHA0274','38VERC2902','38PCHA1006','38PCHA0032','38PCHA0121','38VERC1824','38PCHA0111','38VERC1743','38PCHA1004','38PCHA0928','38PCHA0123','38VERC1848','38PCHA1008','38PCHA0057','38PCHA0065','38PCHA1009','38VERC2008','38BDAU0908','38BELL0154','38VERC2495','38GRES0637','38VERC3451','38VERC0383','38VERC3222','38VERC3348','38GRES0629','38VERC2912','38CHAR0502','38CHAR0045','38VERC2058','38VERC1931','38MATH0010','38CHAR0040','38VERC1924','38GRES0006','38CHAR0900','38VERC2163','38BELL0030','38VERC2100','38GRES0066','38VERC2187','38CHAR0049','38VERC2272','38GRES0077','38VERC2807','38CHAR0079','38VERC2658','38CHAR0066','38CHAR0100','38VERC3008','38BELL0005','38CHAR0903','38CHAR0008','38CHAR0901','38VERC0219','38VERC2414','38VERC2514','38GRES0084','38VERC3076','38GRES0119','38GRES0132','38VERC3311','38CHAR0072','38VERC2718','38CHAR0115','38VERC3205','38VERC3108','38BELL0049','38VERC0215','38VERC2377','38VERC2387','38VERC0216','38BELL0035','38VERC2431','38GRES0080','38VERC2408','38GRES0686','38VERC3623','38VERC2384','38CHAR0468','38MATH0111','38VERC2649','38VERC2661','38BELL0198','38VERC1748','38GRES0437','38GRES0472','38VERC2614','38GRES0462','38VERC2446','38TRIE0215','38VERC2356','38TRIE0211','38VERC2329','38VERC2315','38TRIE0210','38TRIE0222','38VERC2405','38VERC2439','38TRIE0227','38VERC2443','38TRIE0228','38VERC2455','38TRIE0229','38TRIE0022','38CHAR0902','38GRES0141','38VERC3408','38CHAR0113','38VERC3159','38CHAR0001','38CHAR0899','38CHAR0905','38GRES0028','38CHAR0012','38CHAR0906','38CHAR0017','38CHAR0908','38GRES0214','38VERC2141','38VERC3072','38VERC0290','38VERC3376','38GRES0136','38GRES0053','38VERC1668','38GRES0127','38VERC3197','38GRES0183','38GRES1040','38VERC1549','38GRES0045','38GRES0030','38CHAR0907','38GRES1039','38GRES0031','38VERC3399','38BIEV0004','38CHAR0904','38GRES0025','38VERC1533','38CHAR0029','38CHAR0133','38VERC3354','38CHAR0031','38VERC1618','38MATH0083','38VERC1830','38GRES0203','38VERC1858','38MATH0082','38VERC1820','38CHAR0176','38GRES1041','38GRES0251','38VERC2935','38VERC0254','38VERC1926','38VERC3369','38CHAR0135','38VERC3464','38GRES0143','38GRES1038','38GRES0016','38VERC3139','38GRES0278','38VERC2556','38BELL0192','38VERC2513','38CHAR0402','38BELL0164','38BELL0492','38MATH0024','38VERC2487','38TRIE0250','38VERC2646','38TRIE0377','38VERC3523','38BELL0489','38BELL0155','38BELL0491','38MCHA0009','38BELL0162','38BELL0490','38CHAR0034','38VERC1687','38VERC1721','38BELL0025','38BELL0016','38CHAR0909','38MATH0005','38VERC1756','38VERC2641','38BELL0043','38CHAR0027','38CHAR0910','38VERC2601','38GRES0089','38GRES0092','38VERC2629'
s = gpd.pd.Series([
'CHA01','CHA01','CHA17','CHA17','STAP01','STAP01','CHE30','CHE30','STBO03','STBO03','STAN26','STAN26','STAN36','STAN36','STAN34','STAN34','STBO05','STBO05','STBO06','STBO06','STLA039','STLA039','STAN22','STAN22','MON50','MON50','MON56','MON56','MON55','MON55','STLA063','STLA063','STLA070','STLA070',
])
sql = '''
SELECT
v1.site_code,
v1.geom,
v1."source",
v1.rmq_usage_process,
CASE WHEN v1.id_origine IS NULL THEN v2.id_origine ELSE v1.id_origine END id_origine
--v1.id_origine --, v2.geom geom_new
FROM ps.v_pelouseseches v1, ps.v_pelouseseches v2
WHERE v1.geom = v2.geom
AND v1.site_code <> v2.site_code
ORDER BY 3,2;
'''
upd = gpd.read_postgis(sql,con)
upd.drop_duplicates(inplace=True)
upd.rename(columns={'site_code':'id_site'},inplace=True)
upd.loc[upd.source=='PS_CHAMBARAN_CEN38_2013','rmq_usage_process'] = \
upd.loc[upd.source=='PS_CHAMBARAN_CEN38_2013','rmq_usage_process'].replace(["\|",'\/','fasciès'],[" x "," x ",'faciès'],regex=True)
upd = upd[['id_site','rmq_usage_process']]
update_to_sql(upd,con,'r_sites_geom','sites','id_site')
'38MCHA0009','38BELL0491','38BELL0164','38BELL0492','38BELL0154','38VERC2495','38GRES1041','38CHAR0176','38VERC2614','38GRES0472','38VERC2649','38MATH0111','38BELL0198','38VERC2661','38GRES0437','38VERC1748','38MATH0082','38VERC1820','38VERC1830','38MATH0083','38PCHA1087','38PCHA1088','38PCHA1086','38PCHA1089','38VERC1858','38GRES0203','38BDAU0908','38VERC2008','38VERC1926','38VERC0254','38GRES0462','38VERC2446','38VERC2513','38CHAR0402','38BELL0192','38VERC2556','38VERC2384','38CHAR0468','38CHAR0502','38VERC2912','38GRES0214','38VERC2141','38VERC2935','38GRES0251','38VERC3222','38VERC0383','38VERC3072','38VERC0290','38GRES0637','38VERC3451','38VERC3348','38GRES0629','38GRES0278','38VERC3139','38VERC3623','38GRES0686','38BELL0489','38BELL0155','38BDAU0905','38BDAU0907','38BIEV0032','38BIEV0033','38GRES0183','38GRES1040','38BELL0490','38BELL0162','38VERC2629','38GRES0092','38VERC2058','38CHAR0045','38CHAR0115','38VERC3205','38CHAR0901','38CHAR0008','38VERC2431','38BELL0035','38GRES0080','38VERC2408','38VERC0216','38VERC2387','38VERC0215','38VERC2377','38BELL0043','38VERC2641','38CHAR0072','38VERC2718','38CHAR0066','38VERC2658','38CHAR0100','38VERC3008','38BELL0049','38VERC3108','38BELL0005','38CHAR0903','38GRES0119','38VERC3076','38GRES0084','38VERC2514','38CHAR0909','38BELL0016','38VERC3311','38GRES0132','38VERC1687','38CHAR0034','38VERC1756','38MATH0005','38VERC2807','38CHAR0079','38VERC2414','38VERC0219','38BELL0025','38VERC1721','38CHAR0910','38CHAR0027','38GRES0077','38VERC2272','38VERC2187','38CHAR0049','38GRES0066','38VERC2100','38VERC2163','38BELL0030','38GRES0089','38VERC2601','38GRES0006','38CHAR0900','38CHAR0040','38VERC1924','38MATH0010','38VERC1931','38VERC3112','38PCHA0316','38VERC3171','38PCHA0325','38PCHA0906','38PCHA1001','38VERC2732','38PCHA0244','38TRIE0250','38VERC2646','38VERC3399','38BIEV0004','38CHAR0904','38GRES0025','38CHAR0031','38VERC1618','38CHAR0908','38CHAR0017','38GRES0028','38CHAR0905','38TRIE0022','38CHAR0902','38GRES0141','38VERC3408','38CHAR0029','38VERC1533','38VERC3354','38CHAR0133','38GRES0143','38VERC3464','38GRES1038','38GRES0016','38VERC3369','38CHAR0135','38VERC1668','38GRES0053','38VERC3376','38GRES0136','38CHAR0001','38CHAR0899','38CHAR0113','38VERC3159','38VERC1549','38GRES0045','38GRES0127','38VERC3197','38CHAR0907','38GRES0030','38GRES1039','38GRES0031','38CHAR0012','38CHAR0906','38VERC1824','38PCHA0121','38PCHA0928','38PCHA1004','38VERC1848','38PCHA0123','38PCHA0111','38VERC1743','38VERC2487','38MATH0024','38VERC2315','38TRIE0210','38TRIE0211','38VERC2329','38TRIE0215','38VERC2356','38TRIE0222','38VERC2405','38TRIE0377','38VERC3523','38VERC2439','38TRIE0227','38TRIE0229','38VERC2455','38TRIE0228','38VERC2443','38VERC2902','38PCHA0274','38VERC3013','38PCHA0297','38PCHA0286','38VERC2961','38VERC2983','38PCHA0291','38VERC2662','38PCHA0232','38VERC3206','38PCHA0332','38VERC3212','38PCHA0333','38PCHA0326','38VERC3175','38PCHA1003','38PCHA0910','38PCHA1007','38PCHA0044','38VERC3681','38PCHA0474','38PCHA1006','38PCHA0032','38PCHA0018','38PCHA1005','38PCHA0057','38PCHA1008','38PCHA1009','38PCHA0065','38PCHA0908','38PCHA1002'
drop_site = [
'38VERC2141','38VERC3072','38GRES1040','38VERC1830','38VERC1858',''
]
'38VERC2141','38VERC3072','38GRES1040','38VERC1830','38VERC1858','38VERC1820','38GRES1041','38VERC2935','38VERC1926','38VERC3139'
('38VERC2058','38VERC1931','38VERC1924','38CHAR0900','38VERC2163','38VERC2100','38VERC2187','38VERC2272','38VERC2807','38VERC2658','38VERC3008','38CHAR0903','38CHAR0901','38VERC2414','38VERC2514','38VERC3076','38VERC3311','38VERC2718','38VERC3205','38VERC3108','38VERC2377','38VERC2387','38VERC2431','38VERC2408','38CHAR0902','38VERC3408','38VERC3159','38CHAR0899','38CHAR0905','38CHAR0906','38CHAR0908','38VERC3376','38VERC1668','38VERC3197','38VERC1549','38CHAR0907','38GRES1039','38VERC3399','38CHAR0904','38VERC1533','38VERC3354','38VERC1618','38VERC3369','38VERC3464','38GRES1038','38VERC1687','38VERC1721','38CHAR0909','38VERC1756','38VERC2641','38CHAR0910','38VERC2601','38VERC2629')
('38VERC2649','38VERC2661','38VERC1748','38VERC2614','38VERC2446','38VERC2556','38VERC2513','38BELL0492','38BELL0489','38BELL0491','38BELL0490')
'38VERC2495','38VERC2008'
('38VERC2356','38VERC2329','38VERC2315','38VERC2405','38VERC2439','38VERC2443','38VERC2455','38VERC2487','38VERC2646','38VERC3523')
('38VERC3451','38VERC3222','38VERC3348','38VERC2912','38VERC3623','38VERC2384')
'38VERC3681','38PCHA1007','38PCHA1006'
('38VERC3112','38VERC3171','38VERC2662','38VERC2732','38VERC3206','38VERC3013','38VERC2983','38VERC2961','38VERC3212','38VERC3175','38PCHA1005','38VERC2902','38VERC1824','38VERC1743','38VERC1848','38PCHA1008','38PCHA1009')
('38PCHA1087','38PCHA1088')
v_ps[(v_ps.source=='PS_CHAMBARAN_CEN38_2013')& ~(v_ps.auteur_site.isin([
'COSQUER Mélanie (CEN Isère)'
]))].reset_index().site_code.tolist()
path='/home/colas/Documents/9_PROJETS/2_PS/CBNA/Trieves_Vercors/'
file = 'ps_aInserer_for INS1.gpkg'
df = gpd.read_file(path+file)
df.rename_geometry('geom', inplace=True)
df['date'] = '2014-01-01'
df['date'] = gpd.pd.to_datetime(df['date'])
df['structure'] = 'CBNA'
# df.rename(columns={"obs":'auteur'}, inplace=True)
df['auteur'] = 'CBNA'
df.loc[df.statut=="Habitat d'intérêt communautaire",'statut'] = \
"Communautaire"
df.loc[df.statut=="Habitat d'intérêt communautaire retenu prioritaire",'statut'] = \
"Prioritaire"
df.loc[df.statut=="Habitat d'intérêt communautaire, retenu prioritaire",'statut'] = \
"Prioritaire"
df.loc[
df.statut=="""Habitat d'intérêt communautaire, retenu prioritaire pour les sites riches en orchidées""",
'statut'] = "Prioritaire"
df.loc[
df.statut=="""Habitat d'intérêt communautaire retenu prioritaire pour les sites riches en orchidées""",
'statut'] = "Prioritaire"
df.loc[
df.statut=="""Habitat communautaire, retenu prioritaire pour les sites riches en orchidées""",
'statut'] = "Prioritaire"
df.loc[df.statut=="Habitat d'intérêt communautaire retenu prioritaire",'statut'] = \
"Prioritaire"
df.to_file(path+file,driver='GPKG')
'38TRIE1043', '38TRIE1058', '38VERC1189', '38VERC1190','38VERC1191', '38VERC1196', '38VERC1199', '38VERC1200','38VERC1201', '38VERC1202', '38VERC1203', '38VERC1204','38VERC1205', '38VERC1209', '38VERC1210', '38VERC1211','38VERC1212', '38VERC1213', '38VERC1214', '38VERC1216','38VERC1218', '38VERC1227', '38VERC3738', '38VERC3952','38VERC4550', '38VERC4551', '38VERC4552', '38VERC4553','38VERC4554', '38VERC4555', '38VERC4556', '38VERC4557','38VERC4559', '38VERC4560', '38VERC4563', '38VERC4564','38VERC4565', '38VERC4566', '38VERC4568', '38VERC4569','38VERC4570', '38VERC4571', '38VERC4572', '38VERC4573','38VERC4574'
df_maj.loc[df_maj.id_site.isin(['38TRIE0788']),'code_hab2'] = '41.13 (43)'
df = gpd.read_postgis('SELECT * FROM pelouse_seche."PB_codehab_nonPresent_dans_corineBiotope"',pycen.con_bdcen)
df = df.replace(['38.81','31.8 t'],['31.81','31.8122'],regex=True)
df.date = gpd.pd.to_datetime(df.date)
df = df[~df.id_site.isin(lst_error2)]
'38PCHA0291','38PCHA0333','38PCHA0326','38MATH0083','38PCHA0910','38GRES0025','38CHAR0029','38PCHA0111','38GRES0030','38CHAR0012','38GRES0028','38CHAR0027','38GRES0053','38MATH0005','38BELL0030','38BELL0025','38CHAR0049','38MATH0010','38CHAR0079','38GRES0119','38GRES0127','38BELL0049','38BIEV0004','38GRES0006','38CHAR0176','38GRES0031','38GRES0183','38VERC0254','38GRES0214','38VERC0290','38GRES0278','38MATH0082','38GRES0203','38GRES0251','38CHAR0031','38VERC0383','38GRES0136','38GRES0084','38CHAR0135','38CHAR0133','38GRES0045','38GRES0141','38BELL0192','38BDAU0847','38CHAR0402'
update_to_sql(
df[['id_site','remarques']].rename(columns={'remarques':'rmq_interet_patri'}),
con,'r_sites_geom','sites','id_site'
)
df = gpd.read_postgis('SELECT * FROM pelouse_seche."PB_codehabCBNA_nonPresent_dans_corineBiotope"',pycen.con_bdcen)
df = df.replace(['43.171','43.13','31.4B','37.831','52 à 54'],['43','43','31.4','31.831','54'],regex=True)
'38CHAR1238','38CHAR1091','38TRIE2732','38OISA0273','38OISA0272','38GROU0015','38OISA0274','38VERC3785','38TRIE1399','38TRIE1402','38TRIE2443','38TRIE2444','38VERC1467','38VERC1382','38VERC1059','38TRIE0942','38TRIE0788','38OISA0190','38OISA0188','38OISA0189','38VERC0918','38TRIE1010','38TRIE0943','38BVIS0042','38VERC1253','38VERC1322','38VERC1323','38VERC3787'
gpd.read_postgis("SELECT * FROM sites.r_sites_geom WHERE id_site IN ('%s')"%"','".join(DF.id_site),pycen.con)
'38OISA0245','38TRIE1045','38TRIE1165','38TRIE2421','38VERC1401','38VERC1447','38VERC3718','38VERC3766','38VERC4003'
'38VERC4697','38VERC4698','38VERC4699','38VERC4700','38VERC4701','38VERC4704','38VERC4705','38VERC4706','38VERC4707','38VERC4708','38VERC4709','38VERC4710','38VERC4711','38VERC4712','38VERC4713','38VERC4714','38VERC4715','38VERC4716','38VERC4717','38VERC4718','38VERC4719','38VERC4720','38VERC4721','38VERC4722','38VERC4723','38VERC4724','38VERC4726','38VERC4727','38VERC4729','38VERC4730','38VERC4731','38VERC4732','38VERC4733','38VERC4734','38VERC4735','38VERC4736','38VERC4737','38VERC4738','38VERC4739','38VERC4741','38VERC4743','38VERC4746','38VERC4747','38VERC4748'
cvr = gpd.read_postgis("SELECT * FROM ps.v_pelouseseches WHERE site_code IN ('38VERC4740','38VERC4725','38VERC1271','38VERC4739')",pycen.con)
df = gpd.read_postgis("SELECT * FROM ps.v_pelouseseches WHERE site_code IN ('38VERC1249','38VERC1251','38VERC1272')",pycen.con)
tt = gpd.overlay(df,cvr.unary_union,how='difference')
# tt = df.copy()
tt = pycen.tools.Polygons_to_MultiPolygon(tt)
tt.drop(columns=['auteur_geom', 'auteur_attrs'],inplace=True)
tt.drop(columns=['date_geom', 'date_attrs'],inplace=True)
tt.rename(columns={'auteur_site':'auteur'},inplace=True)
tt.rename(columns={'date_site':'date'},inplace=True)
tt.columns = tt.columns.str.replace('cb_hab','code_hab',regex=True)
tt.rename(columns={'site_code':'id_site'},inplace=True)
tt[['auteur','structure']] = tt.auteur.str.split('(',expand=True).sort_index()
tt.auteur = tt.auteur.str.strip()
tt.structure = tt.structure.str.strip().str.replace('\)','',regex=True)
# tt.auteur = 'GRANGE Benjamin'
# tt['structure'] = 'CEN Isère'
'38VERC4697','38VERC4698','38VERC4699','38VERC4700','38VERC4701','38VERC4704','38VERC4705','38VERC4706','38VERC4707','38VERC4708','38VERC4709','38VERC4710','38VERC4711','38VERC4712','38VERC4713','38VERC4714','38VERC4715','38VERC4716','38VERC4717','38VERC4718','38VERC4719','38VERC4720','38VERC4721','38VERC4722','38VERC4723','38VERC4724','38VERC4726','38VERC4727','38VERC4729','38VERC4730','38VERC4731','38VERC4732','38VERC4733','38VERC4734','38VERC4735','38VERC4736','38VERC4737','38VERC4738','38VERC4739','38VERC4741','38VERC4743','38VERC4746','38VERC4747','38VERC4748'
'38VERC4702','38VERC4703','38VERC4725','38VERC4728','38VERC4740','38VERC4742','38VERC4744','38VERC4745'
'38BVIS0047','38VERC4082','38VERC4083','38VERC0618','38VERC1266','38VERC1267','38VERC1232','38VERC1240','38VERC4158','38VERC4158','38VERC4068','38VERC4086','38VERC4083'
'38BVIS0049', '38VERC1231', '38VERC1234', '38VERC1237','38VERC1241', '38VERC1246', '38VERC1247', '38VERC1248','38VERC1250', '38VERC1255', '38VERC1256', '38VERC1257','38VERC1261', '38VERC1262', '38VERC1263', '38VERC1265','38VERC1268', '38VERC1269', '38VERC1271', '38VERC1306','38VERC1322', '38VERC4051', '38VERC4052', '38VERC4056','38VERC4057', '38VERC4084'
3_AZALEE/get_ps_byhab.py
+67
View File
@@ -0,0 +1,67 @@
from pycen import ps,con_bdcen
import geopandas as gpd
# Liste des habitats souhaités
list_habs = ['35.21','35.23','34.12','64.11','64.12']
habs = '|'.join(list_habs)
# v_pelouseseches
dfps = ps.v_pelouseseches()
cb_col = dfps.columns[dfps.columns.str.startswith('cb_hab')]
cdate = dfps.columns[dfps.columns.str.contains('date')]
for c in cdate:
dfps[c] = gpd.pd.to_datetime(dfps[c])
dfps['date'] = dfps[cdate].max(axis=1)
dfps['auteur'] = None
for c in cdate:
typ = c.removeprefix('date')
dfps.loc[dfps[c].eq(dfps.date),'auteur'] = dfps[dfps[c].eq(dfps.date)]['auteur'+typ]
dfps.drop(
columns=[*cdate,*'auteur' + cdate.str.removeprefix('date')],
inplace=True
)
df = gpd.GeoDataFrame()
for c in cb_col:
res = dfps[dfps[c].str.contains(habs,na=False)].copy()
df = gpd.pd.concat([df,res])
df.drop_duplicates(inplace=True)
print('Extract %s unités parcellaires ...'%df.shape[0])
# PB_codehab_nonPresent_dans_corineBiotope
sql = 'SELECT * FROM pelouse_seche."PB_codehab_nonPresent_dans_corineBiotope"'
dfpb = gpd.read_postgis(sql,con_bdcen)
dfps['date'] = gpd.pd.to_datetime(dfps['date'])
dfpb.rename(columns={
'id_site' : 'site_code',
'pratiques':'pratique',
'niv_embrous':'embrouss',
'recouvmnt':'recouvrement',
'remarques':'rmq_site',
'code_hab1':'cb_hab1',
'code_hab2':'cb_hab2',
'code_hab3':'cb_hab3',
'code_hab4':'cb_hab4',
'code_hab5':'cb_hab5',
}, inplace=True)
samecol = dfpb.columns[dfpb.columns.isin(dfps.columns)]
diffcol = dfpb.columns[~dfpb.columns.isin(dfps.columns)]
dfpb = dfpb[samecol]
cb_colpb = dfpb.columns[dfpb.columns.str.startswith('cb_hab')]
list_habspb = ['PS sur sable','Friche sableuse']
habspb = '|'.join(list_habspb)
for c in cb_colpb:
res = dfpb[dfpb[c].str.contains(habspb,na=False)].copy()
df = gpd.pd.concat([df,res])
df.drop_duplicates(inplace=True)
df['date'] = gpd.pd.to_datetime(df['date']).astype(str)
# df['date'] = gpd.pd.to_datetime(df['date'], unit='D')
print('Extract %s unités parcellaires ...'%df.shape[0])
df.to_file('/home/colas/Documents/9_PROJETS/2_PS/EXPORT/PNA_pelouse_sableuse.gpkg',driver='GPKG')
3_AZALEE/gn_pr_zh/trigger_delete.py
+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-.
from pycen import con
# import pycen
drop = '''
DROP TRIGGER IF EXISTS trigger_end_zh ON gn_pr_zh.t_zh;
DROP FUNCTION IF EXISTS zones_humides.end_zh();
'''
s2 = '''CREATE OR REPLACE FUNCTION zones_humides.end_zh()
RETURNS trigger
LANGUAGE plpgsql
AS $function$
BEGIN
IF (TG_OP = 'DELETE') then
UPDATE sites.sites
SET date_fin = current_timestamp
WHERE id = OLD.code;
END IF;
RETURN NULL;
END;
$function$
;'''
s1 = '''CREATE TRIGGER trigger_end_zh BEFORE
DELETE ON
gn_pr_zh.t_zh FOR EACH ROW EXECUTE FUNCTION zones_humides.end_zh()'''
with con.begin() as cnx:
cnx.execute(drop)
cnx.execute(s2)
cnx.execute(s1)
3_AZALEE/gn_pr_zh/trigger_insert.py
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-.
from pycen import con
# import pycen
drop = '''
DROP TRIGGER IF EXISTS trigger_insert_new_zh ON gn_pr_zh.t_zh;
DROP FUNCTION IF EXISTS zones_humides.insert_new_zh();
'''
s2 = '''CREATE OR REPLACE FUNCTION zones_humides.insert_new_zh()
RETURNS trigger
LANGUAGE plpgsql
AS $function$
BEGIN
INSERT into sites.sites (id,nom,date_deb,id_typo_sdage,remarques)
VALUES (new.code,new.main_name,new.create_date,(SELECT cd_nomenclature FROM gn_pr_zh.t_nomenclatures WHERE id = new.id_sdage),NULL);
INSERT into sites.r_sites_geom (id_site,geom,date,id_lot,id_origine)
VALUES (new.code,ST_TRANSFORM(new.geom,2154),new.create_date,NULL,new.zh_uuid::text);
RETURN NULL;
END;
$function$
;'''
s1 = '''CREATE TRIGGER trigger_insert_new_zh
AFTER INSERT OR UPDATE
ON gn_pr_zh.t_zh
FOR EACH ROW
EXECUTE FUNCTION zones_humides.insert_new_zh();
'''
with con.begin() as cnx:
cnx.execute(drop)
cnx.execute(s2)
cnx.execute(s1)
3_AZALEE/import_habCBNA_bdcen38.py
+163
View File
@@ -0,0 +1,163 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Nom : : recup_ps.py
#Description :
#Copyright : 2021, CEN38
#Auteur : Colas Geier
#Version : 1.0
# from _typeshed import IdentityFunction
import pandas as pd
import geopandas as gpd
from sqlalchemy.engine import URL
from sqlalchemy import create_engine
import pycen
# Parametres bdd IN
from pycen import con_bdcen as con
dic = {
'Ssp':'IDFINAL',
'cod_cb':'code_cb',
'ccorineb':'code_cb',
'id_orig':'idorig',
'pourcentag':'part',
'carte_cb':'type',
'lib_cb':'libelle',
'id':'idorig',
'hectare':'hectares',
'nbre_hc':'hectares',
'observat':'obs',
'surf_m2':'surface',
'statut_car':'statutcart',
}
# table = '"PS_CHAMBARAN_CEN38_2013"'
path = '/home/colas/Documents/9_PROJETS/2_PS/CBNA/'
fV = path + 'PNR_Vercors_SHP/PNRV_data_04_2012.dbf'
fgV = path + 'PNR_Vercors_SHP/PNRV_poly_10_2008_L93_region.shp'
nV = 'cr_VERCORS_habitats_CBNA_1999-2007'
fC = path + 'PNR_Chartreuse_SHP/Chartreuse_2012_data.dbf'
fgC = path + 'PNR_Chartreuse_SHP/Chartreuse_2012_poly_region.shp'
nC = 'cr_CHARTREUSE_habitats_CBNA_2000-2012'
fT = path + 'Trieves_Vercors/TRIEVES_DATA.dbf'
fgT = path + 'Trieves_Vercors/TRIEVES_VERCORS.shp'
nT = 'cr_TRIEVES+VERCORS_habitats_CBNA'
fE = path + 'PNE/DELPHINE_DATA.dbf'
fgE = path + 'PNE/DELPHINE_POLY_L93.shp'
nE = 'cr_ECRIN_habitats_CBNA'
fileG = fgV
file = fV
name = nV
gdf = gpd.read_file(fileG)
gdf = pycen.tools.dropZ(gdf,geom_col='geometry')
dbf = gpd.read_file(file)
dbf.rename(columns=dic,inplace=True)
gdf.rename(columns=dic,inplace=True)
if gdf.columns[0] != 'IDFINAL':
gdf.rename(columns={gdf.columns[0]:'IDFINAL'},inplace=True)
# if dbf.columns[0] != 'IDFINAL':
# dbf.rename(columns={dbf.columns[0]:'IDFINAL'},inplace=True)
dbf.sort_values('IDFINAL', inplace=True)
gdf.sort_values('IDFINAL', inplace=True)
dbf.drop(columns=['geometry'], inplace=True)
same_cols = gdf.columns[gdf.columns.isin(dbf.columns)]
gdf.rename_geometry('geom',inplace=True)
dbf.drop_duplicates(inplace=True)
dbf.reset_index(drop=True,inplace=True)
dbf.columns = dbf.columns.str.lower()
gdf.columns = gdf.columns.str.lower()
dbf.rename(columns=dic,inplace=True)
gdf.rename(columns=dic,inplace=True)
dbf.drop(columns=[
'clemax','cle_code','codevegfin','code_final','lettre','hectares','lib_delphi','lien_habit','lien_typol','evolutif_m','codedelphi',
'cb_carte','cb_2','ch_eur25','ch_stat','cb_choix','choix','codedelphine',], inplace=True, errors='ignore')
gdf.drop(columns=[
'surface','hectares','support','echelle','perimetr',
'part','lettre','phyto','code_cb','cd_eur15','statut','type1','type2','cod_mil',], inplace=True, errors='ignore')
dbf.loc[dbf.statut=='Habitat non désigné','statut'] = None
dbf.loc[dbf.statut=='A préciser','statut'] = None
dbf.loc[dbf.code_cb=='A préciser','code_cb'] = None
dbf.loc[dbf.phyto=='A préciser','phyto'] = None
dbf.loc[dbf.phyto=='Néant','phyto'] = None
#### PNE ####
# dbf.loc[dbf.eur25=='Néant','eur25'] = None
# dbf.loc[dbf.l_eur25=='Néant','l_eur25'] = None
# dbf.loc[dbf.code_cb=='Néant','code_cb'] = None
# dbf.part = dbf.part.astype(int).astype(str)
# dbf.sort_values(['idfinal','part'], ascending=[True,False], inplace=True)
# dbf.dropna(axis=0, subset=['code_cb'], inplace=True)
dbf.sort_values(['idfinal','part'],ascending=[True,False],inplace=True)
tt = [list(range(1,dbf[dbf.idfinal==i].shape[0]+1)) for i in dbf.idfinal.unique() ]
dbf['num'] = [item for sublist in tt for item in sublist]
# dbf.rename(columns={'code_eur27':'code_eur27.'}, inplace=True)
dbf.rename(columns={'code_eur15':'code_eur15.'}, inplace=True)
# dbf.rename(columns={'eur25':'code_eur25.'}, inplace=True)
# dbf.rename(columns={'l_eur25':'l_eur25.'}, inplace=True)
gdf.set_index(['idfinal','idorig','secteur'], inplace=True)
dbf.set_index(['idfinal','idorig','secteur'], inplace=True)
# gdf.set_index(['idfinal','idorig'], inplace=True)
# dbf.set_index(['idfinal','idorig'], inplace=True)
dbf.set_index('num',append=True, inplace=True)
dbf2 = dbf.unstack().copy()
dbf2.dropna(axis=1,how='all', inplace=True)
lst_col = dbf2.columns.get_level_values(0).unique()
stt = ['statut%s'%i for i in range(1,dbf2.iloc[:,dbf2.columns.get_level_values(0)=='statut'].shape[1]+1)]
stc = ['statutcart%s'%i for i in range(1,dbf2.iloc[:,dbf2.columns.get_level_values(0)=='statutcart'].shape[1]+1)]
ste = ['site%s'%i for i in range(1,dbf2.iloc[:,dbf2.columns.get_level_values(0)=='site'].shape[1]+1)]
dbf2.columns = [x+str(j) for x,j in dbf2.columns]
for s in stt:
dbf2.loc[dbf2.statut1.isna(),'statut1'] = dbf2.loc[dbf2.statut1.isna(),s]
for s in stc:
dbf2.loc[dbf2.statutcart1.isna(),'statutcart1'] = dbf2.loc[dbf2.statutcart1.isna(),s]
dbf2.drop(columns=[
*stt[1:],*ste[1:],*stc[1:]
],inplace=True)
dbf2.rename(columns={
'statut1':'statut',
'statutcart1':'statutcart',
'site1':'site'
}, inplace=True)
dbf2.loc[dbf2.statut.isna(),'statut'] = 'N.D'
dbf2.part1 = dbf2.part1.astype(int)
dbf2.columns = dbf2.columns.str.replace('code_cb','code_hab',regex=True)
dbf2.columns = dbf2.columns.str.replace('part','n_hab',regex=True)
df = pd.merge(gdf,dbf2,how='inner',right_index=True,left_index=True)
df.geom = df.buffer(0)
df = pycen.tools.Polygons_to_MultiPolygon(df)
df.to_postgis(
name=name,
con=con,
schema='habitat',
if_exists='replace',
index=True,
geom_col='geom',
)
# dbf2[dbf2.columns[dbf2.columns.str.contains('type')]]
# dbf2[dbf2.columns[dbf2.columns.str.contains('phyto')]]
# dbf2[dbf2.columns[dbf2.columns.str.contains('phyto')]][~dbf2.phyto12.isna()]
# dbf2[dbf2.columns[dbf2.columns.str.contains('libelle')]]
# dbf2[dbf2.columns[dbf2.columns.str.contains('libelle')]][~dbf2.libelle13.isna()]
# dbf2[dbf2.columns[dbf2.columns.str.contains('codeveg')]]
# dbf2[dbf2.columns[dbf2.columns.str.contains('codeveg')]][~dbf2.codeveg13.isna()]
# dbf2.iloc[:,dbf2.columns.get_level_values(0)=='type']
# dbf2.iloc[:,dbf2.columns.get_level_values(0)=='statut']
# dbf2.columns = [x+str(j) for x,j in dbf2.columns]
# dbf2.part1 = dbf2.part1.astype(int)
3_AZALEE/insert_ps.py
+1499
View File
File diff suppressed because it is too large Load Diff
3_AZALEE/insert_zh.py
+1041
View File
File diff suppressed because it is too large Load Diff
3_AZALEE/recup_CBNA_habPS.py
+345
View File
@@ -0,0 +1,345 @@
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
from statistics import geometric_mean
import geopandas as gpd
import pandas as pd
from sqlalchemy.engine import URL
from sqlalchemy import create_engine
# from shapely.geometry import Polygon
import pycen
from os import listdir, chdir
from pathlib import Path
from zipfile import ZipFile
from rasterstats import zonal_stats
# Path MNT
path0 = '/home/colas/Documents/9_PROJETS/2_PS/'
path0_mnt = '/home/colas/Documents/9_PROJETS/3_PGZH/'
path = path0_mnt + 'SIG/'
p_mltifct = 'multi_fonctions/IGN - BD Alti 25M/'
path2_mnt = path0+'MNT/'
file_mnt25 = 'MNT_25m.tif'
file_mnt5 = 'MNT_5m.tif'
# Parametres bdd IN
from pycen import con_bdcen as con
# con = create_engine('postgresql+psycopg2://{0}:{1}@{2}/{3}'.format(user,pwd,adr,base), echo=False)
def keepgeom_indept(df):
dnat = pycen.ref.get_districtNat().geom.unary_union
df['in_isere'] = df.intersects(dnat)
df = df[df.in_isere].copy()
df.dropna(axis=1, how='all', inplace=True)
del df['in_isere']
return df
# df = gpd.read_postgis(
# sql = 'SELECT * FROM pelouse_seche."cr_PS_CBNA_habitats_aggreg_06_2020"',
# con = con
# )
lst_tab = ['"cr_VERCORS_habitats_CBNA_1999-2007"','"cr_TRIEVES+VERCORS_habitats_CBNA_2014"','"cr_ECRIN_habitats_CBNA_2014"','"cr_CHARTREUSE_habitats_CBNA_2000-2012"',]
table = lst_tab[3]
df = gpd.read_postgis(
sql = 'SELECT * FROM habitat.%s'%table,
con = con
)
df.set_index('idfinal', inplace=True)
# df.set_index('data_id', inplace=True)
colnhab = df.columns[df.columns.str.startswith('n_hab')]
nhab = df[colnhab].copy()
nhab['SUMM'] = nhab.sum(axis=1)
nhab['SUMM'].unique()
lst_hab = ['34.1','34.3','34.4','34.7','35.2','62.3','64.1']
df2 = pd.DataFrame()
for h in lst_hab:
tmp = df[
(df.code_hab1.str.contains(h,na=False))|
(df.code_hab2.str.contains(h,na=False)&(df.n_hab2 >= 20))|
(df.code_hab3.str.contains(h,na=False)&(df.n_hab3 >= 20))|
(df.code_hab4.str.contains(h,na=False)&(df.n_hab4 >= 20))
]
df2 = pd.concat([df2,tmp])
df3 = df2.copy()
# Before 1385 / After 1422
lst_hab2 = ['41','42','43','44','81','82','83']
for h2 in lst_hab2:
tmp2 = df2[
(df2.code_hab1.str.contains('&'+h2,na=False)&(df2.n_hab1 >= 60))
]
df2.drop(labels=tmp2.index,inplace=True)
for h2 in lst_hab2:
tmp2 = df2[
(df2.code_hab1.str.contains(h2+'.',na=False)&(df2.n_hab1 >= 60))
]
df2.drop(labels=tmp2.index,inplace=True)
for h2 in lst_hab2:
tmp3 = df3[
(df3.code_hab1.str.contains(h2,na=False)&(df3.n_hab1 >= 60))
]
df3.drop(labels=tmp3.index,inplace=True)
df2 = keepgeom_indept(df2)
df3 = keepgeom_indept(df3)
df = df2.copy()
df.reset_index(inplace=True,drop=False)
# ps = pycen.ps.get_sitesGeom()
sql = 'SELECT * FROM ps.v_pelouseseches_all'
PS = gpd.read_postgis(sql,pycen.con)
ps = PS[PS.source==table.replace('"','')].copy()
# 1385
not_in_azalee = df2[~df2.index.isin(df3.index)].copy()
in_azalee = df2[df2.index.isin(df3.index)].copy()
not_in_azalee.to_file(path0+'CBNA/MISSING_DATA_INDB/chartreuse.gpkg',layer='not_in_azalee')
in_azalee.to_file(path0+'CBNA/MISSING_DATA_INDB/chartreuse.gpkg',layer='in_azalee')
ps.columns
# Drop poly who intersect poly non-CBNA
# ps_autre = ps[~ps.auteur_geom.str.contains('CBNA')]
# intersect = gpd.sjoin(df,ps_autre,op='intersects').idfinal.tolist()
# drop_index = df[df.idfinal.isin(intersect)].index
# df.drop(drop_index,inplace=True)
# Drop poly who intersect poly CBNA
# ps_cbna = ps[ps.auteur_geom.str.contains('CBNA')]
# intersect = gpd.sjoin(
# df,
# gpd.GeoDataFrame( geometry=ps_cbna.centroid),
# op='intersects').idfinal.tolist()
# drop_index = df[df.idfinal.isin(intersect)].index
# df.drop(drop_index,inplace=True)
from os.path import join
PATH = '/home/colas/Documents/9_PROJETS/2_PS/CBNA/MISSING_DATA_INDB'
FILE = 'CHARTREUSE.gpkg'
df = gpd.read_file(join(PATH,FILE))
if 'date' not in df.columns and 'annee' in df.columns:
df.loc[~df.annee.isna(),'date'] = df.loc[~df.annee.isna(),'annee'].astype(str) + '-01-01'
df['date'] = pd.to_datetime(df['date'])
colnhab2 = df2.columns[df2.columns.str.startswith('n_hab')]
nhab2 = df2[colnhab2].copy()
nhab2['SUMM'] = nhab2.sum(axis=1)
nhab2['SUMM'].unique()
if table == '"cr_CHARTREUSE_habitats_CBNA_2000-2012"':
dic = {
'JCV':'VILLARET Jean-charles',
'GP' :'PACHE Gilles',
'AM' :'MIKOLAJCZAK Alexis',
'TL' :'LEGLAND Thomas',
'HM' :'MERLE Hugues',
'CC' :'Crassous Claire',
# 'AL'
# 'LA'
# 'JC'
}
df.obs = df.obs.replace(dic)
df.loc[~df.obs.isin(['ONF38',*dic.values()]),'obs'] = 'CBNA'
df.loc[df.structur=='ONF 38','structur'] = 'ONF38'
if table == '"cr_ECRIN_habitats_CBNA_2014"':
df['auteur'] = 'PnE'
df['structure'] = 'PnE'
df['date'] = '2014-01-01'
df['date'] = pd.to_datetime(df['date'])
if table == '"cr_TRIEVES+VERCORS_habitats_CBNA_2014"':
df['auteur'] = 'CBNA'
df['structure'] = 'CBNA'
df['date'] = '2014-01-01'
df['date'] = pd.to_datetime(df['date'])
if table == '"cr_VERCORS_habitats_CBNA_1999-2007"':
df['structure'] = 'CBNA'
if table in [
'"cr_VERCORS_habitats_CBNA_1999-2007"','"cr_TRIEVES+VERCORS_habitats_CBNA_2014"'
]:
df.loc[df.statut=="Habitat d'intérêt communautaire",'statut'] = \
"Communautaire"
df.loc[df.statut=="Habitat d'intérêt communautaire retenu prioritaire",'statut'] = \
"Prioritaire"
df.loc[df.statut=="Habitat d'intérêt communautaire, retenu prioritaire",'statut'] = \
"Prioritaire"
df.loc[
df.statut=="""Habitat d'intérêt communautaire, retenu prioritaire pour les sites riches en orchidées""",
'statut'] = "Prioritaire"
df.loc[
df.statut=="""Habitat d'intérêt communautaire retenu prioritaire pour les sites riches en orchidées""",
'statut'] = "Prioritaire"
df.loc[
df.statut=="""Habitat communautaire, retenu prioritaire pour les sites riches en orchidées""",
'statut'] = "Prioritaire"
df.loc[df.statut=="Habitat d'intérêt communautaire retenu prioritaire",'statut'] = \
"Prioritaire"
if table in [
'"cr_CHARTREUSE_habitats_CBNA_2000-2012"', '"cr_ECRIN_habitats_CBNA_2014"']:
dic = {'PR':'Prioritaire','IC':'Communautaire','NC':'Non communautaire'}
df.statut.replace(dic,inplace=True)
df.reset_index(inplace=True, drop=True)
df.to_postgis(
'ps'+table[3:-1],pycen.con_bdcen,'pelouse_seche',geom_col='geom',if_exists='replace'
)
# [CROSS_MNT]
ddf = df[['idfinal','geom']].copy()
ddf.set_geometry('geom',inplace=True)
# home = str(Path.home())
# chdir(path+p_mltifct)
# Dir = listdir()
# Dir = [x for x in Dir if '.zip' in x]
# # stats = pd.DataFrame()
# for i, d in enumerate(Dir):
# zip = ZipFile(d).namelist()
# z = [z for z in zip if 'MNT' in z][0]
# print(z)
# tmp = zonal_stats(ddf.geom,'/vsizip/{zip}/{mnt}'.format(zip=d,mnt=z),stats="max")
# tmp = pd.DataFrame(tmp)
# tmp.columns = ['max%s'%i]
# if i == 0 :
# stats = tmp
# else:
# stats = pd.merge(stats,tmp,how='left',left_index=True,right_index=True)
# stats['bilan'] = stats.max(axis=1)
# tmp = pd.merge(df,stats['bilan'],left_index=True,right_index=True)
# tmp.bilan = tmp.bilan.round()
# zonal_tmp = zonal_stats(ddf.geom,path2_mnt+file_mnt25,stats="mean max",nodata=0,all_touched=True)
# import rasterio
# with rasterio.open(path2_mnt+file_mnt5) as src:
# affine = src.transform
# array = src.read(1)
# zonal_tmp3 = zonal_stats(ddf, array, affine=affine,stats="mean max",nodata=0,all_touched=True)
# zonal_tmp = pd.DataFrame(zonal_tmp)
# zonal_tmp3 = pd.DataFrame(zonal_tmp3)
# zonal_tmp.columns = ['max_alti','mean_alti']
# zonal_tmp3.columns = ['max_alti','mean_alti']
# tmp = pd.merge(df,zonal_tmp2,left_index=True,right_index=True)
from pycen import con as con_aza
sql = 'SELECT * FROM ps.v_pelouseseches_noalti'
ddf = gpd.read_postgis(sql,con_aza)
zonal_tmp2 = zonal_stats(ddf,path2_mnt+file_mnt5,stats="max",nodata=0,all_touched=True)
zonal_tmp2 = pd.DataFrame(zonal_tmp2)
zonal_tmp2.columns = ['max_alti']
tmp = pd.concat([ddf,zonal_tmp2], axis=1)
tmp = tmp.set_geometry('geom', crs=2154)
tmp['infeq_1200'] = tmp.max_alti <= 1200
tmp2 = tmp[['site_code','infeq_1200']].copy()
tmp2.rename(columns={'site_code':'id_site'},inplace=True)
tmp2.to_sql('r_infeq_1200m',con_aza,'ps',if_exists='append',index=False)
df.to_postgis(
'ps'+table[3:-1],pycen.con_bdcen,'pelouse_seche',geom_col='geom',if_exists='replace'
)
PSinfeq1200 = tmp[tmp.max_alti <= 1200].copy()
PSsup1200 = tmp[tmp.max_alti > 1200].copy()
# PSinfeq1200 = tmp[tmp.bilan <= 1200].copy()
# PSsup1200 = tmp[tmp.bilan > 1200].copy()
# PSinfeq1200.rename(columns={'bilan':'max_alti'}, inplace=True)
# PSsup1200.rename(columns={'bilan':'max_alti'}, inplace=True)
PSinfeq1200.dropna(axis=1, how='all',inplace=True)
if table in [
'"cr_VERCORS_habitats_CBNA_1999-2007"','"cr_TRIEVES+VERCORS_habitats_CBNA_2014"']:
PSinfeq1200.loc[PSinfeq1200.statut=="Habitat d'intérêt communautaire",'statut'] = \
"Communautaire"
PSinfeq1200.loc[PSinfeq1200.statut=="Habitat d'intérêt communautaire retenu prioritaire",'statut'] = \
"Prioritaire"
PSinfeq1200.loc[PSinfeq1200.statut=="Habitat d'intérêt communautaire, retenu prioritaire",'statut'] = \
"Prioritaire"
PSinfeq1200.loc[
PSinfeq1200.statut=="""Habitat d'intérêt communautaire, retenu prioritaire pour les sites riches en orchidées""",
'statut'] = "Prioritaire"
PSinfeq1200.loc[
PSinfeq1200.statut=="""Habitat d'intérêt communautaire retenu prioritaire pour les sites riches en orchidées""",
'statut'] = "Prioritaire"
PSinfeq1200.loc[
PSinfeq1200.statut=="""Habitat communautaire, retenu prioritaire pour les sites riches en orchidées""",
'statut'] = "Prioritaire"
PSinfeq1200.loc[PSinfeq1200.statut=="Habitat d'intérêt communautaire retenu prioritaire",'statut'] = \
"Prioritaire"
if table in [
'"cr_CHARTREUSE_habitats_CBNA_2000-2012"', '"cr_ECRIN_habitats_CBNA_2014"']:
dic = {'PR':'Prioritaire','IC':'Communautaire','NC':'Non communautaire'}
PSinfeq1200.statut.replace(dic,inplace=True)
DF_INI = PSinfeq1200.copy()
df = DF_INI.copy()
# df['structure'] = 'CBNA'
# sql = """
# DELETE FROM sites.sites
# WHERE sites.id in (SELECT id_site from sites.r_sites_geom where id_lot = 1);
# """
# with pycen.con.begin() as cnx:
# cnx.execute(sql)
path0 = '/home/colas/Documents/9_PROJETS/2_PS/TO IMPORT/'
cols_date = PSsup1200.columns[PSsup1200.columns.str.contains('date')]
PSsup1200[cols_date] = PSsup1200[cols_date].astype(str)
PSsup1200.to_file(path0 + 'PS_ECRIN_CBNA_2014_SUP1200.shp')
cols_date = DF_INI.columns[DF_INI.columns.str.contains('date')]
DF_INI[cols_date] = DF_INI[cols_date].astype(str)
DF_INI.to_file(path0 + 'PS_ECRIN_CBNA_2014_INF1200.shp')
op = '''
gdalwarp -overwrite -s_srs EPSG:2154 -t_srs EPSG:2154 -co FORMAT=GPKG -of GTiff -tr 25.0 -25.0 -tap -cutline \
"PG:dbname='bd-cen-38' host=192.168.0.189 port=5432 sslmode=disable user='cgeier' password='adm1n*bdCen'" \
-csql "SELECT * FROM habitat."cr_TRIEVES_VERCORS_habitats_CBNA_2014" WHERE idfinal in (24060)" \
"{mnt}" {out}
'''.format(
mnt=path2_mnt+'out.tif',
out=path2_mnt+'24060.tif')
system(op)
mnt = rio.open(Path_tmp+mnt_out)
sql = 'SELECT * FROM pelouse_seche."PB_codehab_nonPresent_dans_corineBiotope"'
df = gpd.read_postgis(sql,con)
lst = ['cr_TRIEVES+VERCORS_habitats_CBNA_2014','cr_VERCORS_habitats_CBNA_1999-2007','cr_ECRIN_habitats_CBNA_2014','cr_CHARTREUSE_habitats_CBNA_2000-2012']
df = df[df.table_org.isin(lst)]
df.loc[df.auteur=='PnE','structure'] = 'PnE'
sql = 'SELECT * FROM pelouse_seche."PB_codehabCBNA_nonPresent_dans_corineBiotope"'
df2 = gpd.read_postgis(sql,con)
df3 = pd.concat([df,df2])
df3.set_geometry('geom',inplace=True,crs=2154)
df3.to_postgis(
name='PB_codehabCBNA_nonPresent_dans_corineBiotope',
con=con,
schema='pelouse_seche',
if_exists='replace',
geom_col='geom'
)
3_AZALEE/recup_ps.py
+1456
View File
File diff suppressed because it is too large Load Diff
3_AZALEE/refHydro_to_db.py
+118
View File
@@ -0,0 +1,118 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from pycen.tools import Polygons_to_MultiPolygon
from pycen import con
import geopandas as gpd
from sqlalchemy import create_engine, text
from geoalchemy2 import Geometry
user_cad = 'cgeier' # utilisateur de connexion à la bdd
pwd_cad = 'adm1n*bdCen' # mot de passe de la bdd
adr_cad = '91.134.194.221' # adresse ip de la bdd
port_cad = '5432' # port de la bdd
base_cad = 'rhomeo' # nom de la bdd
schema_cad = 'bdtopo3'
con = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user_cad,pwd_cad,adr_cad,port_cad,base_cad), echo=False)
file = '/media/cgeier/Disk2/5_BDD/RPG/1_DONNEES_LIVRAISON_2021/RPG_2-0_SHP_LAMB93_R84_2021-01-01/ILOTS_ANONYMES.shp'
dic = {
'CD_SSBV': 'cdssbv',
'LIB_SSBV': 'nom',
'CD_COMGEO': 'cd_comgeo',
'CD_CTB': 'cd_ctb',
'CD_': 'cd_',
'CD_SUBUNIT': 'cd_subunit',
'geometry': 'geom',
}
name_table = 'ilots_anonymes_2021'
name_schema = 'rpgnew'
# Parametres bdd OUT
# user_zh = 'postgres'
# pwd_zh = 'tutu'
# adr_zh = '192.168.60.10'
# base_zh = 'bd_cen'
# con_zh = create_engine('postgresql+psycopg2://{0}:{1}@{2}/{3}'.format(user_zh,pwd_zh,adr_zh,base_zh), echo=False)
select_cols = list(dic.values())
df = gpd.read_file(file)
# df = df.set_geometry('geom')
df.rename_geometry('geom', inplace=True)
df.columns = df.columns.str.lower()
if not df.crs:
df.set_crs(epsg=2154, inplace=True)
if df.crs.srs != 'epsg:2154':
df.to_crs(epsg=2154, inplace=True)
# df.rename(columns=dic, inplace=True)
# df = df[select_cols]
if 'id' not in df.columns:
df.index.name = 'id'
df.reset_index(drop=False, inplace=True)
if 'Polygon' in df.geom_type.unique() and 'MultiPolygon' in df.geom_type.unique():
df = Polygons_to_MultiPolygon(df)
# from shapely.geometry.multipolygon import MultiPolygon
# tmp = df.loc[df.geom_type == 'Polygon'].copy()
# geom = [MultiPolygon([x]) for x in tmp.loc[tmp.geom_type == 'Polygon','geom']]
# tmp = tmp.set_geometry(geom)
# df = gpd.pd.concat([df.drop(tmp.index), tmp]).sort_values('id').reset_index(drop=True)
if 'LineString' in df.geom_type.unique() and 'MultiLineString' in df.geom_type.unique():
from shapely.geometry.multilinestring import MultiLineString
tmp = df.loc[df.geom_type == 'LineString'].copy()
geom = [MultiLineString([x]) for x in tmp.loc[tmp.geom_type == 'LineString','geom']]
tmp = tmp.set_geometry(geom)
df = gpd.pd.concat([df.drop(tmp.index), tmp]).sort_values('id').reset_index(drop=True)
con.begin()
df.to_postgis(
name = name_table,
con = con,
schema = name_schema,
if_exists = 'replace',
chunksize = 100000,
dtype={'geom':Geometry(srid=2154)}
)
sql = """ALTER TABLE {sch}.{tab} OWNER TO cen_admin;
GRANT ALL ON TABLE {sch}.{tab} TO cen_admin;
GRANT ALL ON TABLE {sch}.{tab} TO grp_admin;
--GRANT SELECT ON TABLE {sch}.{tab} TO grp_consult;
""".format(sch=name_schema,tab=name_table)
with con.begin() as cnx:
cnx.execute(sql)
sql = """
REVOKE SELECT ON TABLE {sch}.{tab} FROM grp_consult;
""".format(sch=name_schema,tab=name_table)
with con.begin() as cnx:
cnx.execute(sql)
if 'rpg' in name_table:
v_tab = 'v_'+name_table.replace('reg','')+'isere'
lst_col = ','.join(tuple('s.'+df.columns))
sql = """CREATE OR REPLACE VIEW {sch}.{v_tab}
AS SELECT {cols}
FROM {sch}.{tab} s,
ref_territoire.dept_isere reg
WHERE st_intersects(s.geom, reg.geom);
ALTER TABLE {sch}.{v_tab} OWNER TO cen_admin;
GRANT ALL ON TABLE {sch}.{v_tab} TO cen_admin;
GRANT ALL ON TABLE {sch}.{v_tab} TO grp_admin;
GRANT SELECT ON TABLE {sch}.{v_tab} TO grp_consult;""".format(sch=name_schema,tab=name_table,v_tab=v_tab,cols=lst_col)
with con.begin() as cnx:
cnx.execute(sql)
3_AZALEE/rpg_to_db.py
+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from pycen import con
import geopandas as gpd
from datetime import datetime as dt
file = '/home/colas/Documents/tmp/rpg/RPG_2-2__SHP_LAMB93_R84_2023-01-01/RPG/1_DONNEES_LIVRAISON_2023/RPG_2-2__SHP_LAMB93_R84_2023-01-01/PARCELLES_GRAPHIQUES.shp'
name_table = 'rpg_parcelles_graphiques_reg'
name_schema = 'ref_territoire'
thisyear = dt.now().year-1
df = gpd.read_file(file)
df.columns = df.columns.str.lower()
# df = df.set_geometry('geom')
df.rename_geometry('geom', inplace=True)
if not df.crs:
df.set_crs(epsg=2154, inplace=True)
if df.crs.srs != 'epsg:2154':
df.to_crs(epsg=2154, inplace=True)
exist_table = con.dialect.get_table_names(con, schema = name_schema)
if name_table in exist_table:
other_table = name_table.replace('rpg','rpg'+str(thisyear-1))
view = 'v_'+name_table.replace('reg','isere')
other_view = view.replace('rpg','rpg'+str(thisyear-1))
sql = """
ALTER TABLE IF EXISTS {sch}.{tab} RENAME TO "{othertab}";
ALTER VIEW IF EXISTS {sch}.{view} RENAME TO "v_{otherview}";
ALTER INDEX IF EXISTS {sch}.idx_rpg_parcelles_graphiques_reg_geom RENAME TO "idx_rpg{old_year}_parcelles_graphiques_reg_geom";
""".format(sch=name_schema,tab=name_table,othertab=other_table,old_year=thisyear-1,view=view,otherview=other_view)
with con.begin() as cnx:
cnx.execute(sql)
df.drop_duplicates().to_postgis(
name = name_table,
con = con,
schema = name_schema,
if_exists = 'replace',
index = False
)
sql = """
ALTER TABLE {sch}.{tab} OWNER TO cen_admin;
GRANT ALL ON TABLE {sch}.{tab} TO cen_admin;
GRANT ALL ON TABLE {sch}.{tab} TO grp_admin;
GRANT SELECT ON TABLE {sch}.{tab} TO grp_consult;
""".format(sch=name_schema,tab=name_table)
comment = """COMMENT ON TABLE {sch}.{tab} IS 'Version {v} 2.1 - Table des parcelles graphiques de la région Auvergne Rhône-Alpes.'""".format(sch=name_schema,tab=name_table,v=thisyear)
with con.begin() as cnx:
cnx.execute(sql)
cnx.execute(comment)
if 'rpg' in name_table:
v_tab = 'v_'+name_table.replace('reg','')+'isere'
lst_col = ','.join(tuple('s.'+df.columns))
sql = """CREATE OR REPLACE VIEW {sch}.{v_tab}
AS SELECT {cols}
FROM {sch}.{tab} s,
ref_territoire.dept_isere reg
WHERE st_intersects(s.geom, reg.geom);
ALTER TABLE {sch}.{v_tab} OWNER TO cen_admin;
GRANT ALL ON TABLE {sch}.{v_tab} TO cen_admin;
GRANT ALL ON TABLE {sch}.{v_tab} TO grp_admin;
GRANT SELECT ON TABLE {sch}.{v_tab} TO grp_consult;""".format(sch=name_schema,tab=name_table,v_tab=v_tab,cols=lst_col)
comment = """COMMENT ON VIEW {sch}.{v_tab} IS 'Version {v} 2.1 - Table des parcelles graphiques de l''isere.'""".format(sch=name_schema,v_tab=v_tab,v=thisyear)
with con.begin() as cnx:
cnx.execute(sql)
cnx.execute(comment)
3_AZALEE/tmp/PNE_extract_zh.py
+32
View File
@@ -0,0 +1,32 @@
from pycen import zh
import geopandas as gpd
import os
def get_intersected(df1,df2):
is_intersect = df1.intersects(df2.unary_union)
return df1[is_intersect].copy()
def format_data(df):
istype_date = df.dtypes == 'datetime64[ns]'
is_date = df.columns.str.contains('date|heure',case=False,regex=True)
col_date = df.columns[istype_date|is_date]
df[col_date] = df[col_date].astype(str)
if __name__ == "__main__":
PATH = '/home/colas/Documents/tmp/PNE'
file_pne = 'fwdcoucheszoneshumidesisroises/PNE_aoa.shp'
zh = zh()
v_zh = zh.v_zoneshumides()
zhp = zh.zh_ponctuelles()
pne = gpd.read_file(os.path.join(PATH,file_pne))
vzh_pne = get_intersected(v_zh,pne)
zhp_pne = get_intersected(zhp,pne)
format_data(vzh_pne)
format_data(zhp_pne)
vzh_pne.drop(['old_code','type_site'],axis=1).to_file(os.path.join(PATH,'PNE_ZH.gpkg'),driver='GPKG',layer='inventaire_zh')
zhp_pne.to_file(os.path.join(PATH,'PNE_ZH.gpkg'),driver='GPKG',layer='zh_ponctuelle')
3_AZALEE/tmp/PS_GAM2026_standardise_data.py
+41
View File
@@ -0,0 +1,41 @@
import geopandas as gpd
from os import path
PATH = '/home/cgeier/Documents/9_PROJETS/2_PS/2026/Anne BELET - couchepelousessches'
anne = gpd.read_file(path.join(PATH,'couche_finale_modif_CEN.gpkg')).set_index('site_code')
mosa = gpd.read_file(path.join(PATH,'couche_finale_modif_mosaique.gpkg')).set_index('site_code')
call = gpd.read_file(path.join(PATH,'couche_finale2.gpkg')).set_index('site_code')
date_cols = anne.columns[anne.columns.str.contains('date')]
for tab in [anne,mosa,call]:
for d in date_cols :
tab[d] = (gpd.pd.to_datetime(tab[d],format='mixed',utc=True)
.astype(str)
.str.split(' ', expand=True)[0]
)
anne.loc[anne.auteur_attrs=='Anne BELET (GAM)','organisme'] = 'Grenoble Alpes Métropole'
anne.loc[anne.auteur_geom=='Anne BELET (GAM)','organisme'] = 'Grenoble Alpes Métropole'
anne.loc[anne.auteur_geom=='Ag Mosaique Env','organisme'] = 'Mosaïque Environnement'
anne.loc[anne.auteur_geom=='Ag Mosaique Env','auteur_geom'] = 'Mosaïque Environnement'
anne.loc[anne.auteur_attrs=='Anne BELET (GAM)','auteur'] = 'BELET Anne'
anne.loc[anne.auteur_geom=='Anne BELET (GAM)','auteur'] = 'BELET Anne'
anne.loc[anne.auteur_geom=='Mosaïque Environnement','auteur'] = 'Mosaïque Environnement'
anne['id_origine'] = anne.index.str[:10]
anne['id_site'] = anne.index.copy()
anne.loc[anne.index.str.len()<10,'id_site'] = None
anne.loc[anne.index.str.len()>10,'id_site'] = None
anne.rename(columns={
'rmq_interet_patri':'remarques','rmq_usage_process':'legende'
}, inplace=True)
anne.to_file(path.join(PATH,'GAM2026_couche_finale_modif_CEN (corrige).gpkg'),index=False)
mosa.to_file(path.join(PATH,'GAM2026_couche_finale_modif_mosaique (corrige).gpkg'))
call.to_file(path.join(PATH,'GAM2026_couche_finale (corrige).gpkg'))
anne.to_excel(path.join(PATH,'couche_finale_modif_CEN.xlsx'),index=False)
mosa.to_excel(path.join(PATH,'couche_finale_modif_mosaique.xlsx'))
3_AZALEE/tmp/compare_inv2021_actu2025.py
+36
View File
@@ -0,0 +1,36 @@
import pandas as pd
import geopandas as gpd
from pycen import con_gn
old = gpd.read_file('/home/cgeier/Téléchargements/Telechargement_1772622390_2696/data/n_zone_humide_s_038/n_zone_humide_s_038.json')
new = gpd.read_postgis('SELECT * FROM gn_exports.v_synthese_zones_humides_isere',con_gn).to_crs(2154)
tab = pd.read_csv('/home/cgeier/Documents/9_PROJETS/1_ZH/2024/RENDU SIG/pv_evolution_dossier.csv',sep=',')
for etude in tab.nom_etude.unique():
if tab[(tab.nom_etude==etude)].organisme_transmission.any():
continue
print('\n',etude)
news = tab[(tab.nom_etude==etude) & (tab.status_inventaire == 'Nouvelle')]
modif = tab[(tab.nom_etude==etude) & (tab.status_inventaire== 'Modification')]
supp = tab[(tab.nom_etude==etude) & (tab.status_inventaire == 'Supression')]
new_add = new[new.code.isin(news.code_zh.tolist())].copy()
del_zh = old[old.id_local.isin(supp.code_zh.tolist())]
print('\tNouvelles surfaces ajoutées : %f ha'%(round(new_add.area.sum()/10000,2)),' ; %i zh'%new_add.shape[0],' ; \n\tcode : ',', '.join(list(new_add.code)))
print('\tSurfaces supprimées : %f ha'%(round(del_zh.area.sum()/10000,2)),' ; %i zh'%del_zh.shape[0],' ; \n\tcode : ',', '.join(list(del_zh.id_local)))
code_modif = modif.code_zh.tolist()
diff = (new[new.code.isin(code_modif)]
.sort_values('code').set_index('code').area - old[old.id_local.isin(code_modif)]
.sort_values('id_local').set_index('id_local').area
)
add = diff[diff > 0]
red = diff[diff < 0]
print('\tModification surfaces ajoutées : %f ha'%(round(add.sum()/10000,2)),' ; %i zh'%add.shape[0],' ; \n\tcode : ',', '.join(list(add.index)))
print('\tModification surfaces perdues : %f ha'%(round(red.sum()/10000,2)),' ; %i zh'%red.shape[0],' ; \n\tcode : ',', '.join(list(red.index)))
3_AZALEE/tmp/correct&maj_idsite_ps.py
+116
View File
@@ -0,0 +1,116 @@
import pycen
import geopandas as gpd
dstct = pycen.ref.get_districtNat()
ps = pycen.ps.get_sitesGeom(statut='all')
ps['org_bd'] = ps.id_site.str[2:6]
ps['num'] = ps.id_site.str[6:]
df = gpd.sjoin(ps,dstct[['abrev','geom']]).drop(columns=['index_right'])
df.sort_values('id_site',inplace=True)
tmp = pycen.tools.calc_recouvrmt(df1=df[df.org != df.abrev], df2=dstct)
from geopandas import sjoin
tmp = sjoin(
df1,
df2[['geom']],
op = 'intersects',
how = 'left')
tmp.dropna(subset=['index_right'],inplace=True)
tmp.index_right = tmp.index_right.astype(int)
tmp.reset_index(inplace=True)
tmp = tmp.join(
df2[['geom']].rename(columns={'geom': 'right_geom'}),
on=['index_right'], how='left')
tmp2 = tmp[['index_right','right_geom']].copy() \
.rename(columns={'right_geom': 'geom'}) \
.set_geometry('geom')
tmp1 = tmp[['id_site','geom']].copy() \
.set_geometry('geom')
if not tmp1.geom.values.is_valid.all():
tmp1.loc[~tmp1.geom.values.is_valid,'geom'] = tmp1.loc[~tmp1.geom.values.is_valid,'geom'].buffer(0)
if not tmp2.geom.values.is_valid.all():
tmp2.loc[~tmp2.geom.values.is_valid,'geom'] = tmp2.loc[~tmp2.geom.values.is_valid,'geom'].buffer(0)
tmp['perc_rcvmt'] = (tmp1.intersection(tmp2).area/tmp1.area)*100
tmp = tmp.groupby(['id_site']).sum().reset_index()
df1 = df1.merge(tmp[['id_site','perc_rcvmt']], on=['id_site'], how='left')
df1.perc_rcvmt.fillna(0, inplace=True)
df1.perc_rcvmt = df1.perc_rcvmt.round(2)
tmp1['area_rcvmt'] = [gpd.overlay(tmp1.iloc[[i]],tmp2.iloc[[i]]).area.round()[0] for i in range(0,tmp1.shape[0]) ]
# Maj origine district
df = ps[ps.org_bd!=ps.org].sort_values('id_site').copy()
pycen.update_to_sql(
df[['id_site','org']].rename(columns={'id_site':'id'}).drop_duplicates(),
pycen.con,'sites','sites','id'
)
# Redéfinition des numéro de sites
ps_sit = pycen.ps.get_sitesInfos(statut='all')
ps_sit.loc[ps_sit.id.isin(df.id_site),'num'] = None
# attribution de numéro non utilisé
for o in ps_sit.org.unique():
tmp_sit = ps_sit.loc[ps_sit.org==o,'num'].dropna().astype(int)
min = tmp_sit.min()
max = tmp_sit.max()
lst = [str(i) for i in range(min,max) if i not in [*tmp_sit]]
tt = (ps_sit.org==o)&(ps_sit.num.isna())
tmp_sitna = ps_sit[tt]
if tmp_sitna.shape[0] <= len(lst):
ps_sit.loc[tt,'num'] = lst[:tmp_sitna.shape[0]]
else:
ps_sit.loc[(ps_sit.index.isin(tt[tt].index[:len(lst)])),'num'] = lst
tt2 = (ps_sit.org==o)&(ps_sit.num.isna())
min2 = max + 1
max2 = max + 1 + ps_sit[tt2].shape[0]
lst2 = [str(i) for i in range(min2,max2)]
ps_sit.loc[tt2,'num'] = lst2
df2 = ps_sit.loc[ps_sit.id.isin(df.id_site)]
df2.loc[df2.num.str.len() == 1,'num'] = '000'+ df2.loc[df2.num.str.len() == 1,'num']
df2.loc[df2.num.str.len() == 2,'num'] = '00' + df2.loc[df2.num.str.len() == 2,'num']
df2.loc[df2.num.str.len() == 3,'num'] = '0' + df2.loc[df2.num.str.len() == 3,'num']
pycen.update_to_sql(
df2[['id','num']].drop_duplicates(),
pycen.con,'sites','sites','id'
)
# Redéfinition des écritures
ps_sitnew = pycen.ps.get_sitesInfos(id_site=[*df.id_site],statut='all')
ps_sitnew = ps_sitnew[[
'id','dept','org','num',
]]
ps_sitnew['idnew'] = ps_sitnew.dept+ps_sitnew.org+ps_sitnew.num
# incrémentation de l'historique
ps_sitnew[['id','idnew']]\
.rename(columns={
'id':'id_site_old',
'idnew':'id_site_new'
}).to_sql(
'hist_id_site',
pycen.con,'sites',if_exists='append',index=False
)
# modification des noms
pycen.update_to_sql(
ps_sitnew[['index','dept','org','num']].rename(columns={'index':'id'}).drop_duplicates(),
pycen.con,'sites','sites',['dept','org','num']
)
pycen.update_to_sql(
ps_sitnew[['idnew','dept','org','num']].rename(columns={'idnew':'id'}).drop_duplicates(),
pycen.con,'sites','sites',['dept','org','num']
)
3_AZALEE/tmp/hab_aggreg_cbna_2024.py
+233
View File
@@ -0,0 +1,233 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
# 2024-11-26
# LE script permet d'intégrer de nouvelles géométries
# dans la table sites.r_sites_geom. Les AUTEURS ont été
# intégrés manuellement.
import geopandas as gpd
from pycen import con, zh as ZH
from pycen.tools import dropZ, Polygons_to_MultiPolygon
from os import path
from shapely import wkb
from matplotlib import pyplot as plt
import pycen
from sqlalchemy import create_engine
from sqlalchemy.engine import URL
# isin_bdd = True
# # Parametres bdd CEN38 OUT
user = 'cen_admin'
pwd = "#CEN38@venir"
adr = '91.134.194.221'
base = 'azalee_restore'
url = URL.create("postgresql+psycopg2", username=user, password=pwd, host=adr, database=base)
con = create_engine(url)
zh = ZH()
r_geom = zh.get_sitesGeom()
r_geom = gpd.read_postgis('''
SELECT r_sites_geom.* FROM sites.r_sites_geom
JOIN sites.sites ON sites.id = r_sites_geom.id_site AND sites.id_type_milieu = 1
''',con)
r_geom.dropna(how="all",axis=1,inplace=True)
r_geom.drop(columns=['id','date_insert','id_origine','id_lot'],inplace=True)
r_geom.sort_values(['id_site','date'],inplace=True)
r_geom.drop_duplicates(['id_site'],keep='last',inplace=True)
PATH = '/home/colas/Documents/9_PROJETS/1_ZH/MAJ/Actu 2024/FAIT/CBNA/zh38'
file = 'hab_agreg.gpkg'
df = gpd.read_file(path.join(PATH,file))
df.rename_geometry('geom', inplace=True)
df.geom = [wkb.loads(wkb.dumps(g,output_dimension=2)) for g in df.geom]
df.dropna(how='all',axis=1,inplace=True)
# gpd.GeoSeries([r_geom.unary_union],crs=2154).contains(df.geom)
# df.contains(r_geom.geom).any()
# res = df.within(r_geom.unary_union)
perc = 80
zh_caract = df[df.zh=='caractéristique'].copy()
zh_caract['surfzhall'] = zh_caract.groupby(['idpolyfinal'])['n06rechab'].transform('sum')
lst_idpoly = zh_caract[zh_caract.surfzhall >= perc].idpolyfinal.unique()
# zh_caract.within(r_geom.unary_union)
# zh_caract.contains(r_geom.unary_union).any()
# Non Intersection Après 2015 - tout milieux
no_inters = df[
(df.lien_zh == 'aucun') & (df.n07anneehab > 2015) #& (df.zh=='caractéristique')
& (df.idpolyfinal.isin(lst_idpoly))
& (~df.n05lbhab.str.contains('invasif',na=False,case=False))
].copy()
no_inters = no_inters.drop(columns=['id_site']).sjoin(r_geom[['id_site','geom']])
no_inters.to_file(path.join(PATH,'inters.gpkg'),driver='GPKG',layer='no_inters%i'%perc)
len(no_inters.id_site.unique())
# Intersection Après 2015 - tout milieux
inters = df[
(df.lien_zh == 'intersecte') & (df.n07anneehab > 2015) #& (df.zh=='caractéristique')
# & (df.n06rechab.astype(int)>45)
& (df.idpolyfinal.isin(lst_idpoly))
& (~df.n05lbhab.str.contains('invasif',na=False,case=False))
].copy()
inters = inters.drop(columns=['id_site']).sjoin(r_geom[['id_site','geom']])
inters.to_file(path.join(PATH,'inters.gpkg'),driver='GPKG',layer='inters%i'%perc)
len(inters.id_site.unique())
# Intersection Après 2009 - milieux Forêts uniquement
inters_foret = df[
(df.lien_zh == 'intersecte') & (df.zh=='caractéristique') #& (df.n07anneehab > 1999)
# & (df.n06rechab.astype(int)>45)
& (df.idpolyfinal.isin(lst_idpoly)) & (df.n05lbhab.str.contains('for.t',na=False,case=False))
& (~df.n05lbhab.str.contains('invasif',na=False,case=False))
& (~df.idpolyfinal.isin(inters.idpolyfinal.tolist()))
].copy()
inters_foret = inters_foret.drop(columns=['id_site']).sjoin(r_geom[['id_site','geom']])
inters_foret.to_file(path.join(PATH,'inters.gpkg'),driver='GPKG',layer='inters_forets')
# Fusion Géométries
intersF = gpd.pd.concat([inters,inters_foret])
intersF.to_csv(path.join(PATH,'intersection_finale_hab_zh.csv'))
rgeo = (r_geom[r_geom.id_site.isin(intersF.id_site.unique())]
.copy())
for id_site in rgeo.id_site.unique():
rgeo.loc[rgeo.id_site==id_site,'geom'] = gpd.GeoSeries([
gpd.GeoSeries(
[*rgeo[rgeo.id_site==id_site].geom,
*intersF[intersF.id_site==id_site].geom],
crs=2154).unary_union
],
crs=2154,
index=rgeo[rgeo.id_site==id_site].index)
rgeo.date = '2024-01-21'
# Filtre geométries plus grandes
rgeo2 = rgeo[
rgeo.sort_values('id_site').area
> r_geom[r_geom.id_site.isin(rgeo.id_site.unique())].sort_values('id_site').area
]
# Check habitats describes
sql = 'SELECT * FROM zones_humides.r_site_habitat WHERE valid IS True and id_site IN {lst}'.format(lst=tuple(intersF.id_site.unique()))
hab_bdd = gpd.pd.read_sql(sql,con)
insert_hab = gpd.pd.DataFrame()
for id_site in intersF.id_site.unique():
lst_bddhab = hab_bdd[hab_bdd.id_site==id_site].id_cb.tolist()
cbn_hab = intersF[(intersF.id_site==id_site)&(~intersF.n24cdcb.astype(str).isin(lst_bddhab))]
insert_hab = gpd.pd.concat([insert_hab,cbn_hab])
ins_hab = (insert_hab[['n03observat','n04organism','n07anneehab','n24cdcb','id_site']]
.rename(columns={'n24cdcb':'id_cb','n03observat':'observer','n04organism':'organisme','n07anneehab':'annee'})
.replace(
{'AMODEI T':'AMODEI Thomas',
'Lo Parvi':'LO PARVI',
'ACER CAMPESTRE':'Acer Campestre',
'Réflex environnement':'REFLEX Environnement',
'Office National des Forets':'Office National des Forêts',
'Conservatoire botanique national alpin':'Conservatoire Botanique National Alpin',
'Mosaique Environnement, M. Voirin':'Mosaïque Environnement',
'Boucard E.':'BOUCARD E.',
'MALET, A., JOUD. D., LINOSSIER, T.':'MALET A., JOUD Didier, LINOSSIER T.',
'AURAND T., TAIN C':'AURAND Theo, TAIN C.',
'FOLCHER C.':'FOLCHER Caroline',
"Conservatoire d'espaces naturels": 'CEN Isère'
})
.drop_duplicates())
ins_hab['auteur'] = None
ins_hab.loc[ins_hab.observer.isna(),['auteur']] = ins_hab[ins_hab.observer.isna()].organisme
ins_hab.loc[~ins_hab.observer.isna(),['auteur']] = (ins_hab[~ins_hab.observer.isna()].observer +
' (' +ins_hab[~ins_hab.observer.isna()].organisme + ')')
ins_hab['date'] = ins_hab.annee.astype(str) + '-01-01'
ins_hab.drop(columns=['observer','organisme','annee'],inplace=True)
# Insertion geom BDD
rgeo2.to_postgis('r_sites_geom',pycen.con,'sites',if_exists='append',index=False)
siteid = gpd.pd.read_sql('''
SELECT DISTINCT ON (id_site) id,id_site
FROM sites.r_sites_geom
WHERE id_site IN {lst}
ORDER BY id_site,id DESC
'''.format(lst=tuple(rgeo2.id_site.unique())),pycen.con)
usr = siteid.drop(columns=['id_site']).copy()
usr.rename(columns={'id':'id_geom_site'},inplace=True)
usr['id_auteur'] = 96
usr.to_sql('r_geomsites_auteur',pycen.con,schema='sites',if_exists='append',index=False)
# Insertion habitat BDD
siteidgeo = gpd.pd.read_sql('''
SELECT DISTINCT ON (id_site) id id_geom_site,id_site
FROM sites.r_sites_geom
WHERE id_site IN {lst}
ORDER BY id_site,id DESC
'''.format(lst=tuple(ins_hab.id_site.unique())),pycen.con)
ins_habF = ins_hab.merge(siteidgeo, on='id_site',how='left')
ins_habF.sort_values(['id_site','id_cb','date'],inplace=True)
ins_habF.drop_duplicates(subset=['id_site','id_cb'],keep='last',inplace=True)
ins_habF['valid'] = True
ins_habF.to_csv(path.join(PATH,'habs_inserted.csv'))
(ins_habF
.drop(columns=['auteur'])
.to_sql('r_site_habitat',pycen.con,schema='zones_humides',if_exists='append',index=False))
refpers = (pycen.pers.get_auteur2()
.replace({'nom_prenom':{
'ONF38 (ONF)':'Office National des Forêts',
'CBNA (CBNA)':'Conservatoire Botanique National Alpin',
'LO PARVI (LO PARVI)':'LO PARVI',
'CEN Isère (CEN Isère)':'CEN Isère',
'Ecosphère (Ecosphère)':'Ecosphère',
'BURGEAP (BURGEAP)':'BURGEAP',
'Acer Campestre (Acer Campestre)':'Acer Campestre',
'REFLEX Environnement (REFLEX Environnement)':'REFLEX Environnement',
'REFLEX Environnement (REFLEX Environnement)':'REFLEX Environnement'
}})
.replace({'nom_prenom':{
'CBNA':'Conservatoire Botanique National Alpin',
'ONF':'Office National des Forêts',
'BÉGUIN Lucile':'BEGUIN Lucile',
'CEN AURA':'Conservatoire Régional des Espaces Naturels Rhône-Alpes',
'CD Isère':"Département de l'Isère"
}},regex=True))
pers_dict = dict(zip(refpers.nom_prenom,refpers.index))
siteidhab = gpd.pd.read_sql('''
SELECT DISTINCT id,id_site,id_cb,date,id_geom_site
FROM zones_humides.r_site_habitat
WHERE (id_site,id_cb,date,id_geom_site) IN {lst}
ORDER BY id_site,id DESC
'''.format(lst=tuple(tuple(ins_habF[['id_site','id_cb','date','id_geom_site']].itertuples(index=False,name=None)))),pycen.con)
hab_auth = (ins_habF[['id_site','auteur','id_cb','date']]
.astype(str)
.merge(siteidhab.astype(str), on=['id_site','id_cb','date'],how='left')
.rename(columns={'id':'id_sitehab'})
.set_index(['id_sitehab']))
hab_auth['orga'] = (hab_auth.auteur.str.split('(').str[-1]
.str.split(')').str[0])
test_auth = hab_auth.auteur.str.contains(',')
hab_auth1 = (hab_auth[~test_auth].auteur
.replace(pers_dict)
.to_frame('id_auteur'))
hab_auth_tmp = (hab_auth[test_auth]
.auteur.str.split('(').str[0]
.str.split(',')
.explode()
.str.strip()
.to_frame('auteur')
.merge(hab_auth.loc[test_auth,['orga']],left_index=True,right_index=True))
hab_auth2 = ((hab_auth_tmp.auteur+' ('+hab_auth_tmp.orga+')')
.replace({
'BURGEAP (BURGEAP, ECOSPHERE)':'BURGEAP',
'ECOSPHERE (BURGEAP, ECOSPHERE)':'Ecosphère',})
.replace(pers_dict)
.to_frame('id_auteur'))
(gpd.pd.concat([hab_auth1,hab_auth2])
.reset_index(drop=False)
.to_sql('r_rsitehab_auteur',pycen.con,schema='zones_humides',if_exists='append',index=False))
3_AZALEE/tmp/identify_modif_actuZH2024.py
+153
View File
@@ -0,0 +1,153 @@
from pycen import zh as ZH
import geopandas as gpd
import pandas as pd
from sqlalchemy import create_engine
from sqlalchemy.engine import URL
import os
zh = ZH()
# isin_bdd = True
# # Parametres bdd CEN38 OUT
from pycen import con
lst_table = [
'r_sites_geom',
'r_site_critdelim',
'r_site_fctecosociopatri',
'r_site_habitat',
'r_site_reghydro',
'r_site_sub',
'r_site_type_connect',
'r_site_usageprocess',
]
result = {}
for table in lst_table:
sql_old = "SELECT max(t.id) FROM {sch}.{tab} t".format(tab=table,sch='sites' if table == 'r_sites_geom' else 'zones_humides')
if table == 'r_sites_geom':
sql_old += ' JOIN sites.sites ON sites.id = t.id_site AND sites.id_type_milieu = 1'
# else:
# sql_old += ''' JOIN sites.sites ON sites.id = r_sites_geom.id_site AND sites.id_type_milieu = 1'''
id_old = pd.read_sql(sql_old,con)
sql = "SELECT t.* FROM {sch}.{tab} t".format(tab=table,sch='sites' if table == 'r_sites_geom' else 'zones_humides')
if table == 'r_sites_geom':
sql += ' JOIN sites.sites ON sites.id = t.id_site AND sites.id_type_milieu = 1'
sql += " WHERE t.id > {id} AND id_site != '26PNRV0208'".format(id=id_old.values[0][0])
result[table] = pd.read_sql(sql,zh.con)
if table == 'r_sites_geom':
sql_lot = "SELECT * FROM sites.lots WHERE id_lot IN {id}".format(id=tuple(result[table].id_lot.dropna().unique()))
lot = pd.read_sql(sql_lot,zh.con).dropna(how='all',axis=1)
result[table] = result[table].merge(lot, on='id_lot',how='left')
r_geo = result['r_sites_geom']
sql = '''
with lots as (
SELECT DISTINCT ON (id_site)
id_site,lots.*
FROM sites.r_sites_geom
LEFT JOIN sites.lots USING (id_lot)
ORDER BY id_site, "date" DESC
)
SELECT v.*, lots.libelle FROM zones_humides.v_zoneshumides v
JOIN lots ON v.site_code = lots.id_site
WHERE v.site_code IN {lst_code}
'''.format(lst_code=str(tuple(r_geo.id_site.unique())))
vzh = gpd.read_postgis(sql,zh.con).dropna(subset=['libelle']).dropna(how='all',axis=1)
vzh_old = gpd.read_postgis('SELECT * FROM zones_humides.v_zoneshumides',con).dropna(how='all',axis=1)
vzh_new = gpd.read_postgis('SELECT * FROM zones_humides.v_zoneshumides',zh.con).dropna(how='all',axis=1)
date_cols = vzh.columns[vzh.columns.str.contains('date|heure')]
vzh[date_cols] = vzh[date_cols].astype(str)
vzh_old[['old_code',*date_cols]] = vzh_old[['old_code',*date_cols]].astype(str)
vzh_new[['old_code',*date_cols]] = vzh_new[['old_code',*date_cols]].astype(str)
PATH = '/home/colas/Documents/9_PROJETS/1_ZH/2024'
vzh_old.to_file(os.path.join(PATH,'20250415_avancement.gpkg'),driver='GPKG',layer='Inventaire_initiale')
vzh_new.to_file(os.path.join(PATH,'20250415_avancement.gpkg'),driver='GPKG',layer='Inv_actualisee_20250415')
for _lib in vzh.libelle.unique():
lib = (_lib.removesuffix('2024')
.removeprefix('ZH')
.replace('actualisation','CENIsere'))
tmp = vzh[vzh.libelle == _lib]
new = tmp[~tmp.site_code.isin(vzh_old.site_code.tolist())]
upt = tmp[tmp.site_code.isin(vzh_old.site_code.tolist())]
if not new.empty:
print(f'NEW {lib} : ',str(new.site_code.tolist()))
# new.to_file(os.path.join(PATH,'20250415_avancement.gpkg'),driver='GPKG',layer='newzh'+lib+str(new.shape[0]))
if not upt.empty:
print(f'UPDT {lib} : ',str(upt.site_code.tolist()))
# upt.to_file(os.path.join(PATH,'20250415_avancement.gpkg'),driver='GPKG',layer='majzh'+lib+str(upt.shape[0]))
vzh_reduc = (vzh[vzh.site_code.isin(vzh_old.site_code.tolist())].sort_values('site_code').set_index('site_code').area.sub(vzh_old[vzh_old.site_code.isin(vzh.site_code.tolist())].sort_values('site_code').set_index('site_code').area,fill_value=0))
vzh_reduc[vzh_reduc<0].sum() / 10000
vzh_reduc[vzh_reduc>0].sum() / 10000
area_maj = (vzh[vzh.site_code.isin(vzh_old.site_code.tolist())].area.sum() - vzh_old[vzh_old.site_code.isin(vzh.site_code.tolist())].area.sum()) / 10000
area_new = vzh[~vzh.site_code.isin(vzh_old.site_code.tolist())].area.sum() / 10000
cbna = vzh[vzh.libelle=='ZH_CBNA_2024']
new_cbna = cbna[(~cbna.site_code.isin(vzh_old.site_code.tolist()))&(vzh.libelle=='ZH_CBNA_2024')].area.sum() / 10000
maj_cbna = (cbna[cbna.site_code.isin(vzh_old.site_code.tolist())].area.sum() - vzh_old[vzh_old.site_code.isin(cbna.site_code.tolist())].area.sum()) / 10000
pgszh = vzh[vzh.libelle=='ZH_pgszhBelledonne_2024']
new_pgszh = pgszh[(~pgszh.site_code.isin(vzh_old.site_code.tolist()))&(vzh.libelle=='ZH_pgszhBelledonne_2024')].area.sum() / 10000
maj_pgszh = (pgszh[pgszh.site_code.isin(vzh_old.site_code.tolist())].area.sum() - vzh_old[vzh_old.site_code.isin(pgszh.site_code.tolist())].area.sum()) / 10000
evin = vzh[vzh.libelle=='ZH_EVINERUDE_2024']
new_evin = evin[(~evin.site_code.isin(vzh_old.site_code.tolist()))&(vzh.libelle=='ZH_EVINERUDE_2024')].area.sum() / 10000
maj_evin = (evin[evin.site_code.isin(vzh_old.site_code.tolist())].area.sum() - vzh_old[vzh_old.site_code.isin(evin.site_code.tolist())].area.sum()) / 10000
lst_attrs = []
r_geo = result['r_sites_geom']
for table in result.keys():
if table == 'r_sites_geom':
continue
df = result[table]
no_geo = df[~df.id_site.isin(r_geo.id_site.unique())]
lst_attrs += no_geo.id_site.unique().tolist()
lst_attrs = [x for x in list(set(lst_attrs)) if x is not None]
r_crit = result['r_site_critdelim']
r_fct = result['r_site_fctecosociopatri']
r_hab = result['r_site_habitat']
r_hyd = result['r_site_reghydro']
r_sub = result['r_site_sub']
r_connect = result['r_site_type_connect']
r_usg = result['r_site_usageprocess']
new_r_crit = r_crit[r_crit.id_site.isin(lst_attrs)]
new_r_fct = r_fct[r_fct.id_site.isin(lst_attrs)]
new_r_hab = r_hab[r_hab.id_site.isin(lst_attrs)]
new_r_hyd = r_hyd[r_hyd.id_site.isin(lst_attrs)]
new_r_sub = r_sub[r_sub.id_site.isin(lst_attrs)]
new_r_connect = r_connect[r_connect.id_site.isin(lst_attrs)]
new_r_usg = r_usg[r_usg.id_site.isin(lst_attrs)]
len_new_crit = new_r_crit.shape[0]
len_new_fct = new_r_fct.shape[0]
len_new_hab = new_r_hab.shape[0]
len_new_hyd = new_r_hyd.shape[0]
len_new_sub = new_r_sub.shape[0]
len_new_connect = new_r_connect.shape[0]
len_new_usg = new_r_usg.shape[0]
lst_new_crit = new_r_crit.id_site.unique().tolist()
lst_new_fct = new_r_fct.id_site.unique().tolist()
lst_new_hab = new_r_hab.id_site.unique().tolist()
lst_new_hyd = new_r_hyd.id_site.unique().tolist()
lst_new_sub = new_r_sub.id_site.unique().tolist()
lst_new_connect = new_r_connect.id_site.unique().tolist()
lst_new_usg = new_r_usg.id_site.unique().tolist()
tmp = gpd.read_file('/home/colas/Documents/9_PROJETS/1_ZH/MAJ/Actu 2024/BE EPODE - LEZE/KIT INVENTAIRE38 MODIFIE/a envoyer prêt/GEOM/38CG0105_Epode2025.gpkg')
tmp_old = gpd.read_postgis('SELECT * FROM zones_humides.v_zoneshumides WHERE site_code = \'38CG0105\'',con)
3_AZALEE/tmp/maj_geom_belledo.py
+20
View File
@@ -0,0 +1,20 @@
#!/usr/bin/env python3
# -*- coding:UTF-8 -*-
from pycen import con, update_to_sql
import geopandas as gpd
file = "/media/cgeier/SRV/FICHIERS/OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/ETUDES/PGSZH_Belledonne/Inventaire 2023/ZH_Belledonne_MAJ_2023_NB.shp"
df = gpd.read_file(file)
df = (df[df.site_code.isin(['38BB0028','38BB0055','38RD0170'])]
.filter(items=['site_code', 'geometry']))
df.rename_geometry('geom', inplace=True)
df.rename(columns={'site_code': 'id_site'}, inplace=True)
df_updt = df[df.id_site.isin(['38RD0170'])]
update_to_sql(df_updt,con,'r_sites_geom','sites','id_site')
df_ins = df[df.id_site.isin(['38BB0028','38BB0055'])]
df_ins['date'] = '2023-06-21'
df_ins.to_postgis('r_sites_geom',con,if_exists='append',index=False,schema='sites')
3_AZALEE/tmp/manip_invPS_CDIsère2016.py
+146
View File
@@ -0,0 +1,146 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
import geopandas as gpd
from pycen import con
path0 = '/home/colas/Documents/9_PROJETS/2_PS/TO IMPORT/'
file = 'CDisère_PS2016_4Montagnes.shp'
dic = {
'identifian':'ident',
'habitat1' : 'code_hab1',
'partpoly1': 'n_hab1',
'habitat2' : 'code_hab2',
'partpoly2': 'n_hab2',
'habitat3' : 'code_hab3',
'partpoly3': 'n_hab3',
'habitat4' : 'code_hab4',
'partpoly4': 'n_hab4',
'légende':'legende',
'embrou_cod':'niv_embrous',
'embrous':'%_embrous',
'sol nu_cod':'recouvmnt',
'sol nu':'%_recouvmnt',
'orchidées':'rmq_phyto',
'alliance':'rmq_phytosocio',
'maec':'rmq_contract_agri',
}
dic_auth = {
'Anne-Lyse Murro':'MURRO Anne-Lyse',
'Lucas Berardi' : 'BERARDI Lucas'
}
dic_prat = {
'Mixte' : 'Fauche, Pâturage',
'Adandon':'Abandon',
'P�turage':'Pâturage'
}
dic_stat = {
'Aucun' : 'Non communautaire',
'Prioriataire' : 'Prioritaire'
}
dic_date = {
'mai 2016' : '2016-05-01',
'juin 2016' : '2016-06-01',
'juillet 2016' : '2016-07-01',
'aout 2016' : '2016-08-01',
'août 2016' : '2016-08-01',
}
keep_cols = ['auteur','date',*dic.values(),'rosette','pratique','statut','remarques','geom']
def emb_correct(df):
l1 = ['GIR3','AU7']
l2 = ['BOI1','VIL10','VIL8a']
l3 = ['MEA4a','ME']
l4 = ['MO3','MA3','VIL14','BD1','VF','CHA2','CHA5','ROM3']
l5 = ['BEL1','JA5','AU4']
df.loc[df.index.isin(l1),'niv_embrous'] = 1
df.loc[df.index.isin(l2),'%_embrous'] = 4
df.loc[df.index.isin(l3),'%_embrous'] = 29
df.loc[df.index.isin(l4),'%_embrous'] = 30
df.loc[df.index.isin(l5),'%_embrous'] = 60
return df
def sol_correct(df):
# df.loc[df.index=='GEY1','%_recouvmnt'] = 19
df.loc[df.index=='CHA5','%_recouvmnt'] = 21
# df.loc[df.index=='GEY2','%_recouvmnt'] = 19
# df.loc[df.index=='SN1','%_recouvmnt'] = 19
df.loc[df.index=='AU1','%_recouvmnt'] = 21
df.loc[(df.recouvmnt==1)&(df['%_recouvmnt']>0),'%_recouvmnt'] = 0
df.loc[(df.recouvmnt==2)&(df['%_recouvmnt']>10),'%_recouvmnt'] = 10
df.loc[(df.recouvmnt==3)&(df['%_recouvmnt']>20),'%_recouvmnt'] = 20
df.loc[(df.recouvmnt==4)&(df['%_recouvmnt']<21),'%_recouvmnt'] = 21
return df
# récupération des données
sql = 'SELECT site_code,geom FROM ps.v_pelouseseches'
v_ps = gpd.read_postgis(sql, con)
df0 = gpd.read_file(path0 + file)
df0.columns = df0.columns.str.lower()
df0.rename(columns=dic, inplace=True)
df0.rename_geometry('geom', inplace=True)
df = df0[keep_cols].copy()
df.set_index('ident', inplace=True)
df.pratique.replace(dic_prat,inplace=True)
df.statut.replace(dic_stat,inplace=True)
df.auteur.replace(dic_auth,inplace=True,regex=True)
df.date = df.date.str.lower()
df.date.replace(dic_date,inplace=True)
df.date = gpd.pd.to_datetime(df.date)
df = emb_correct(df)
df = sol_correct(df)
# Correction code Corine Biotope
df.loc[df.code_hab2=='41D','code_hab2'] = '41.D'
# Concaténation des remarques
df.loc[~df.rmq_phyto.isna(),'remarques'] = df.loc[~df.rmq_phyto.isna(),'remarques'] +\
'; rmq_phyto : ' + df.loc[~df.rmq_phyto.isna(),'rmq_phyto']
df.loc[~df.rosette.isna(),'remarques'] = df.loc[~df.rosette.isna(),'remarques'] +\
'; rosette : ' + df.loc[~df.rosette.isna(),'rosette']
df.loc[~df.rmq_phytosocio.isna(),'remarques'] = df.loc[~df.rmq_phytosocio.isna(),'remarques'] +\
'; rmq_phytosocio : ' + df.loc[~df.rmq_phytosocio.isna(),'rmq_phytosocio']
df.loc[~df.rmq_contract_agri.isna(),'remarques'] = df.loc[~df.rmq_contract_agri.isna(),'remarques'] +\
'; rmq_contract_agri : ' + df.loc[~df.rmq_contract_agri.isna(),'rmq_contract_agri']
del df['rmq_phyto']
del df['rosette']
del df['rmq_phytosocio']
del df['rmq_contract_agri']
df.reset_index(drop=False,inplace=True)
df.rename(columns={'ident':'id_origine'}, inplace=True)
df['table_org'] = file
df['structure'] = 'CD Isère'
# END SCRIPT #
## tri des géométries avant insertion
# Identification des superpositions new_site/old_site
df_inters = gpd.sjoin(df,v_ps, how='left')
del df_inters['index_right']
lst_old_site = df_inters.site_code.unique()
v_ps = v_ps[v_ps.site_code.isin(lst_old_site)].copy()
# Calcul du recouvrement
df.reset_index(drop=False,inplace=True)
v_ps.loc[:,'surf'] = v_ps.area
tmp = gpd.overlay(v_ps,df[['ident','geom']],how='intersection',make_valid=True)
tmp['perc_rcvmt'] = 100*tmp.area/tmp['surf']
tmp_sitemaj = tmp[(tmp.perc_rcvmt > 10)&(~tmp.ident.duplicated(keep=False))].ident
tmp_sitenew = tmp[tmp.perc_rcvmt <= 10].ident.unique()
site_newvsold = {
'old' : [*tmp.site_code.unique()],
'new' : [*tmp.ident.unique()],
}
df_new = df_inters[df_inters.site_code.isna()]
df_inters.drop(df_new.index,inplace=True)
df_maj = df_inters[~df_inters.site_code.duplicated(keep=False)]
df_inters.drop(df_maj.index,inplace=True)
df_rep = df_inters.copy()
df_new = gpd.pd.concat([df_new,df_rep],axis=0)
3_AZALEE/tmp/manip_invPS_PS_AGGREGATION_NB_AG_2022.py
+70
View File
@@ -0,0 +1,70 @@
import geopandas as gpd
import os
import pycen
import numpy as np
path = '/home/colas/Documents/9_PROJETS/2_PS/TO IMPORT'
file = 'PS_AGGREGATION_NB_AG.shp'
def split_auth(col):
return col.str.split(r' \(',expand=True).replace('\)','',regex=True)
def format_auth(col):
auth = pycen.pers.get_auteur()
for c in col.unique():
tmp = c.lower().split(' ')
cd1 = (np.array([tmp[0] == n for n in auth.nom.str.lower()]) |
np.array([tmp[0] == n for n in auth.prenom.str.lower()]))
cd2 = (np.array([tmp[1] == n for n in auth.nom.str.lower()]) |
np.array([tmp[1] == n for n in auth.prenom.str.lower()]))
r = auth[cd1&cd2].nom_prenom.values[0]
col.replace(c,r,inplace=True)
return col
if __name__ == "__main__":
df = gpd.read_file(os.path.join(path,file))
df.columns = df.columns.str.lower()
df.dropna(axis=1,how='all', inplace=True)
for c in df.columns:
if df[c].dtype == object:
df[c] = df[c].str.strip()
if 'date' in c :
df[c] = gpd.pd.to_datetime(df[c])
n_hab = df.columns[df.columns.str.startswith('n_hab')]
df[df[n_hab].sum(axis=1) != 100]
# Auteurs
df[['auteur_split','structure']] = split_auth(df.auteur)
df.auteur_split = format_auth(df.auteur_split)
del df['auteur']
df.rename(columns={'auteur_split':'auteur'},inplace=True)
# Pratiques
df.pratiques.replace([r' \/ '],r', ',regex=True,inplace=True)
df.pratiques.replace({
'paturage broyage':'pâturage, broyage',
'abandon paturage':'abandon, pâturage',
'paturage':'pâturage'
}, inplace=True)
df.type_patur.replace({'N.D':None
}, inplace=True)
cd3 = (df.pratiques=='pâturage')&(~df.type_patur.isna())
df.loc[cd3,'pratiques'] = df.loc[cd3,'pratiques'] +' '+ df.loc[cd3,'type_patur']
df.loc[cd3,'type_patur'] = None
cd4 = (~df.pratiques.isin(['pâturage','pâturage ?']))&(df.pratiques.str.contains('pâturage'))&(~df.type_patur.isna())
df.loc[cd4,'pratiques'] = df.loc[cd4,'pratiques'].str.split(', ',expand=True)[0] + ' ' + df.loc[cd4,'type_patur'] + ', ' + df.loc[cd4,'pratiques'].str.split(', ',expand=True)[1]
df.loc[cd4,'type_patur'] = None
df.pratiques.replace({
'pâturage caprin - ovin':'pâturage caprin, pâturage ovin',
'jardin':'tonte',
'pâturage anin' : 'pâturage asin',
'pâturage lama' : 'pâturage asin',
'abandon ?' : 'incertain',
'pâturage ?' : 'incertain'
}, inplace=True)
3_AZALEE/tmp/manip_invPS_platière.py
+65
View File
@@ -0,0 +1,65 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
import pandas as pd
import geopandas as gpd
import pycen
path0 = '/home/colas/Documents/9_PROJETS/2_PS/TO IMPORT/'
path = 'cartohabitatsplatiere/'
file = 'Habitats I33 2008.gpkg'
df = gpd.read_file(path0+path+file)
df.rename_geometry('geom', inplace=True)
if file == 'Habitats I33 2008.gpkg' :
df.columns = df.columns.str.lower()
df.rename(columns={'date_maj':'date'}, inplace=True)
df['remarques'] = \
'détails : ' + df['type_détaillé'] + ';' +\
' regroupement : ' + df['type_regroupé'] + ';'\
' code_n2000 : ' + df['code_n2000'] + ';'\
' age : ' + df['age'] + ';'
del df['id']
del df['code_enjeu']
del df['enjeu_numérique']
del df['complexe_hab']
del df['echelle']
del df['entretien']
del df['type_détaillé']
del df['type_regroupé']
del df['surface']
del df['num_site']
del df['code_n2000']
del df['age']
# df.set_index('num_site',inplace=True)
df['auteur'] = 'PONT Bernard'
df['structure'] = "AAIP"
lst_hab = ['34.1','34.3','34.4','34.7','35.2','62.3','64.1']
df = df[df.code_cb.str.contains('|'.join(lst_hab))]
lst_hab2 = ['41','42','43','44','81','82','83']
df = df[~df.code_cb.str.contains('|'.join(lst_hab2))]
df.replace({'IC':'Communautaire'}, inplace=True)
df.index.name = 'ident'
df.reset_index(inplace=True, drop=False)
df.rename(columns={'code_cb':'code_hab1'}, inplace=True)
df['n_hab1'] = 100
ps = pycen.ps.get_sitesGeom()
# nv = gpd.read_file(path0+'PS_nature_vivante_2014.shp')
# nv.geometry = nv.buffer(50)
intersect = gpd.sjoin(df,ps,op='intersects')['ident'].tolist()
drop_index = df[df['ident'].isin(intersect)].index
df.drop(drop_index,inplace=True)
df.reset_index(inplace=True, drop=True)
# df.to_file(path0+path+'Habitats I33 2008 (manquants).gpkg', driver='GPKG')
# df = gpd.read_file(path0+path+'Habitats I33 2008 (manquants).gpkg')
3_AZALEE/tmp/manip_invZH_Huez.py
+9
View File
@@ -0,0 +1,9 @@
import geopandas as gpd
import os
if __name__ == "__main__":
from pycen import con
PATH = "/home/colas/Documents/9_PROJETS/1_ZH/MAJ/GERECO - Schéma_strat_ZH_Huez"

Some files were not shown because too many files have changed in this diff Show More