CEN38/Python_scripts
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init depot renew

Browse Source
This commit is contained in:
Colas
2026-09-22 11:58:57 +02:00
commit e9fa6a7309
205 changed files with 39721 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
0_FONCIER/CADASTRE/agregg_cadastre.py
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from sqlalchemy.sql import text
from sqlalchemy.engine import URL
from sqlalchemy import create_engine
import pandas as pd
import geopandas as gpd
from pycen import con_cad
# pg_dump -h 91.134.194.221 -d cadastre -U cgeier --schema="38_202207" -s > 38_202207.sql
# psql -h 91.134.194.221 -U cgeier -d cadastre -a -f 38_202207.sql
lst_sch = con_cad.dialect.get_schema_names(con_cad)
lst_sch = ['07_202207','26_202207','42_202207']
for s in lst_sch:
lst_tab = con_cad.dialect.get_table_names(con_cad,s)
for t in lst_tab:
lst_col = con_cad.dialect.get_columns(con_cad,t,s)
lst_col = [x['name'] for x in lst_col ]
if 'geom' in lst_col:
df = gpd.read_postgis('SELECT * FROM "{sch}".{tab}'.format(sch=s,tab=t),con_cad)
if df.empty:
continue
df.to_postgis(t,con_cad,'xx_202207',if_exists='append',index=False)
else :
df = pd.read_sql_table(t,con_cad,s)
if df.empty:
continue
df.to_sql(t,con_cad,'xx_202207',if_exists='append',index=False)
print("{sch}.{tab}".format(sch=s,tab=t))
0_FONCIER/CADASTRE/transfert_cadastre.py
+11
View File
@@ -0,0 +1,11 @@
# RESTORE AU NOM DE LA BDD ORIGINE
# pg_dump -C -h localhost -U localuser dbname | psql -h remotehost -U remoteuser dbname
# pg_dump -C -h 172.17.0.2 -U postgres -d postgres --schema="07_202107" | psql -h 91.134.194.221 -U cgeier -d bd_cen
# RESTORE AU NOM DE LA BDD CIBLE
# pg_dump -C -h 172.17.0.2 -U postgres -d postgres --schema="*_202501" --format=custom | pg_restore -h 91.134.194.221 -U cgeier --dbname="cadastre"
# export PGPASSWORD="xxxxxx";pg_dump -C -h localhost -U cgeier -d cadastre --schema="*_202603" | psql -h 91.134.194.221 -U cgeier --dbname="bd_cen" -W
# export PGPASSWORD="xxxxxx";pg_dump -C -h 91.134.194.221 -U cgeier -d cadastre --schema="*_202603" --format=custom | pg_restore -h 91.134.194.221 -U cgeier --dbname="bd_cen"
0_FONCIER/CADASTRE/view_cadastre.py
+70
View File
@@ -0,0 +1,70 @@
from pycen import con_fon
view_typprop_x = '''
DROP VIEW IF EXISTS {sch}.{view};
CREATE OR REPLACE VIEW {sch}.{view}
AS
SELECT
p.par_id,
p.codcom,
p.ccopre,
p.ccosec,
p.dnupla,
p.dparpi,
p.dcntpa,
p.ccocomm,
p.ccoprem,
p.ccosecm,
p.dnuplam,
p.type,
p.geom,
typprop_lib,
typprop,
array_agg(dnuper) dnuper,
array_agg(ddenom) ddenom
FROM "{sch}".parcelles{_cen} p
LEFT JOIN "{sch}".d_typprop USING (typprop_id)
LEFT JOIN "{sch}".lots{_cen} USING (par_id)
LEFT JOIN "{sch}".cadastre{_cen} USING (lot_id)
LEFT JOIN "{sch}".cptprop{_cen} USING (dnupro)
LEFT JOIN "{sch}".r_prop_cptprop{_cen} USING (dnupro)
LEFT JOIN "{sch}".proprios{_cen} USING (dnuper)
GROUP BY 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
'''
grant = """
GRANT ALL ON TABLE {sch}.{view} TO grp_admin;
GRANT SELECT ON TABLE {sch}.{view} TO grp_consult;
"""
with con_fon.begin() as cnx:
cnx.execute(view_typprop_x.format(sch='cadastre', _cen= '', view='v_prop_parcelles'))
cnx.execute(view_typprop_x.format(sch='cadastre', _cen= '_cen', view='v_prop_parcelles_cen'))
cnx.execute(grant.format(sch='cadastre', view='v_prop_parcelles'))
cnx.execute(grant.format(sch='cadastre', view='v_prop_parcelles_cen'))
view_natcult_x = '''
DROP VIEW IF EXISTS {sch}.{view};
CREATE OR REPLACE VIEW {sch}.{view}
AS
SELECT
p.par_id,
p.geom,
array_agg(d_dsgrpf.natcult_ssgrp) natcult_ssgrp,
array_agg(d_cgrnum.natcult_grp) natcult_grp,
array_agg(d_cnatsp.natcult_natspe) natcult_natspe
FROM "{sch}".parcelles{_cen} p
LEFT JOIN "{sch}".lots{_cen} USING (par_id)
LEFT JOIN "{sch}".lots_natcult{_cen} USING (lot_id)
LEFT JOIN "{sch}".d_dsgrpf USING (dsgrpf)
LEFT JOIN "{sch}".d_cgrnum USING (cgrnum)
LEFT JOIN "{sch}".d_cnatsp USING (cnatsp)
GROUP BY par_id
'''
with con_fon.begin() as cnx:
cnx.execute(view_natcult_x.format(sch='cadastre', _cen= '', view='v_natcult'))
cnx.execute(view_natcult_x.format(sch='cadastre', _cen= '_cen', view='v_natcult_cen'))
cnx.execute(grant.format(sch='cadastre', view='v_natcult'))
cnx.execute(grant.format(sch='cadastre', view='v_natcult_cen'))
0_FONCIER/EXTRANET/create_siteTable.py
+239
View File
@@ -0,0 +1,239 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from pycen import con_fon
d_typinfosite = '''
CREATE TABLE IF NOT EXISTS sites.d_typinfosite (
typinfosite_id serial NOT NULL,
typinfosite_lib varchar(50) NOT NULL COLLATE pg_catalog."default",
CONSTRAINT d_typinfosite_pkey PRIMARY KEY (typinfosite_id)
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS sites.d_typinfosite OWNER to grp_sig;
GRANT SELECT ON TABLE sites.d_typinfosite TO cen_user;
GRANT DELETE, INSERT, UPDATE ON TABLE sites.d_typinfosite TO grp_foncier;
GRANT ALL ON TABLE sites.d_typinfosite TO grp_sig;
'''
ins_typinfosite = '''
INSERT INTO sites.d_typinfosite (typinfosite_lib) VALUES
('Essentiel'),
('Historique'),
('Patrimoine'),
('Objectifs'),
('Enjeux')
;
'''
r_sites_infos = '''
DROP TABLE IF EXISTS sites.r_sites_infos;
CREATE TABLE IF NOT EXISTS sites.r_sites_infos (
infosite_id serial NOT NULL,
site_id varchar(10) NOT NULL,
typinfosite_id integer NOT NULL,
site_info text,
maj_user text,
maj_date varchar(20),
CONSTRAINT r_sites_infos_pkey PRIMARY KEY (infosite_id),
CONSTRAINT r_sites_infos_site_id_fkey FOREIGN KEY (site_id)
REFERENCES sites.sites (site_id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION,
CONSTRAINT r_sites_infos_typinfosite_id_fkey FOREIGN KEY (typinfosite_id)
REFERENCES sites.d_typinfosite (typinfosite_id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS sites.r_sites_infos OWNER to grp_sig;
GRANT SELECT ON TABLE sites.r_sites_infos TO cen_user;
GRANT DELETE, INSERT, UPDATE ON TABLE sites.r_sites_infos TO grp_foncier;
GRANT ALL ON TABLE sites.r_sites_infos TO grp_sig;
CREATE INDEX IF NOT EXISTS fki_r_sites_infos_infosite_id_fkey
ON sites.r_sites_infos USING btree
(infosite_id ASC NULLS LAST)
TABLESPACE pg_default;
CREATE INDEX IF NOT EXISTS fki_r_sites_infos_site_id_fkey
ON sites.r_sites_infos USING btree
(site_id ASC NULLS LAST)
TABLESPACE pg_default;
CREATE INDEX IF NOT EXISTS fki_r_sites_infos_typinfosite_id_fkey
ON sites.r_sites_infos USING btree
(typinfosite_id ASC NULLS LAST)
TABLESPACE pg_default;
'''
with con_fon.begin() as cnx:
cnx.execute(d_typinfosite)
cnx.execute(ins_typinfosite)
cnx.execute(r_sites_infos)
secteurs = '''
CREATE TABLE IF NOT EXISTS sites.secteurs (
id_secteur serial NOT NULL,
site_id varchar(10) NOT NULL,
secteur_nom text NOT NULL COLLATE pg_catalog."default",
CONSTRAINT secteurs_pkey PRIMARY KEY (id_secteur),
CONSTRAINT secteurs_site_id_fkey FOREIGN KEY (site_id)
REFERENCES sites.sites (site_id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS sites.secteurs OWNER to grp_sig;
GRANT SELECT ON TABLE sites.secteurs TO cen_user;
GRANT DELETE, INSERT, UPDATE ON TABLE sites.secteurs TO grp_foncier;
GRANT ALL ON TABLE sites.secteurs TO grp_sig;
CREATE INDEX IF NOT EXISTS fki_secteurs_site_id_fkey
ON sites.secteurs USING btree
(site_id ASC NULLS LAST)
TABLESPACE pg_default;
'''
d_typpartenaire = '''
DROP TABLE IF EXISTS sites.d_typpartenaire;
CREATE TABLE IF NOT EXISTS sites.d_typpartenaire (
typpartenaire_id serial NOT NULL,
typpartenaire_lib text NOT NULL,
CONSTRAINT d_typpartenaire_pkey PRIMARY KEY (typpartenaire_id)
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS sites.d_typpartenaire OWNER to grp_sig;
GRANT SELECT ON TABLE sites.d_typpartenaire TO cen_user;
GRANT DELETE, INSERT, UPDATE ON TABLE sites.d_typpartenaire TO grp_foncier;
GRANT ALL ON TABLE sites.d_typpartenaire TO grp_sig;
'''
ins_typpartenaire = '''
INSERT INTO sites.d_typpartenaire (typpartenaire_lib) VALUES
('Financeurs'),
('Partenaires')
;
'''
r_sites_partenaires = '''
CREATE TABLE IF NOT EXISTS sites.r_sites_partenaires (
partenaire_id serial NOT NULL,
site_id varchar(10) NOT NULL,
typpartenaire_id integer NOT NULL,
maj_user text,
maj_date varchar(20),
CONSTRAINT r_sites_partenaires_pkey PRIMARY KEY (partenaire_id,site_id,typpartenaire_id),
CONSTRAINT r_sites_partenaires_site_id_fkey FOREIGN KEY (site_id)
REFERENCES sites.sites (site_id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION,
CONSTRAINT r_sites_partenaires_typpartenaire_id_fkey FOREIGN KEY (typpartenaire_id)
REFERENCES sites.d_typpartenaire (typpartenaire_id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS sites.r_sites_partenaires OWNER to grp_sig;
GRANT SELECT ON TABLE sites.r_sites_partenaires TO cen_user;
GRANT DELETE, INSERT, UPDATE ON TABLE sites.r_sites_partenaires TO grp_foncier;
GRANT ALL ON TABLE sites.r_sites_partenaires TO grp_sig;
CREATE INDEX IF NOT EXISTS fki_r_sites_partenaires_site_id_fkey
ON sites.r_sites_partenaires USING btree
(site_id ASC NULLS LAST)
TABLESPACE pg_default;
'''
d_typactnotsite = '''
CREATE TABLE IF NOT EXISTS sites.d_typactnotsite (
typactnotsite_id varchar(6) NOT NULL,
typactnotsite_lib varchar(10) NOT NULL,
CONSTRAINT d_typactnotsite_pkey PRIMARY KEY (typactnotsite_id)
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS sites.d_typactnotsite OWNER to grp_sig;
GRANT SELECT ON TABLE sites.d_typactnotsite TO cen_user;
GRANT DELETE, INSERT, UPDATE ON TABLE sites.d_typactnotsite TO grp_foncier;
GRANT ALL ON TABLE sites.d_typactnotsite TO grp_sig;
'''
actions_sites = '''
CREATE TABLE IF NOT EXISTS sites.actions_sites (
action_site_id serial NOT NULL,
action_site_date varchar(20) NOT NULL,
action_site_datemef varchar(20) NOT NULL,
typactnotsite_id varchar(6) NOT NULL,
action_site_lib text NOT NULL,
CONSTRAINT actions_sites_pkey PRIMARY KEY (action_site_id),
CONSTRAINT r_sites_partenaires_typactnotsite_id_fkey FOREIGN KEY (typactnotsite_id)
REFERENCES sites.d_typactnotsite (typactnotsite_id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS sites.actions_sites OWNER to grp_sig;
GRANT SELECT ON TABLE sites.actions_sites TO cen_user;
GRANT DELETE, INSERT, UPDATE ON TABLE sites.actions_sites TO grp_foncier;
GRANT ALL ON TABLE sites.actions_sites TO grp_sig;
'''
r_sites_actions = '''
CREATE TABLE IF NOT EXISTS sites.r_sites_actions (
action_site_id serial NOT NULL,
site_id varchar(10) NOT NULL,
maj_user text,
maj_date varchar(20),
actif varchar(1) NOT NULL,
CONSTRAINT r_sites_actions_pkey PRIMARY KEY (action_site_id,site_id),
CONSTRAINT r_sites_actions_site_id_fkey FOREIGN KEY (site_id)
REFERENCES sites.sites (site_id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION,
CONSTRAINT r_sites_actions_action_site_id_fkey FOREIGN KEY (action_site_id)
REFERENCES sites.actions_sites (action_site_id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS sites.r_sites_actions OWNER to grp_sig;
GRANT SELECT ON TABLE sites.r_sites_actions TO cen_user;
GRANT DELETE, INSERT, UPDATE ON TABLE sites.r_sites_actions TO grp_foncier;
GRANT ALL ON TABLE sites.r_sites_actions TO grp_sig;
'''
notes_sites = '''
CREATE TABLE IF NOT EXISTS sites.notes_sites (
note_site_id serial NOT NULL,
site_id varchar(10) NOT NULL,
note_site_date varchar(20) NOT NULL,
observateur_id integer NOT NULL,
typactnotsite_id integer NOT NULL,
note_site_lib text NOT NULL,
maj_user text,
maj_date varchar(20),
CONSTRAINT notes_sites_pkey PRIMARY KEY (note_site_id),
CONSTRAINT notes_sites_site_id_fkey FOREIGN KEY (site_id)
REFERENCES sites.sites (site_id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION
)
TABLESPACE pg_default;
ALTER TABLE IF EXISTS sites.notes_sites OWNER to grp_sig;
GRANT SELECT ON TABLE sites.notes_sites TO cen_user;
GRANT DELETE, INSERT, UPDATE ON TABLE sites.notes_sites TO grp_foncier;
GRANT ALL ON TABLE sites.notes_sites TO grp_sig;
'''
with con_fon.begin() as cnx:
cnx.execute(d_typactnotsite)
cnx.execute(actions_sites)
cnx.execute(r_sites_actions)
0_FONCIER/EXTRANET/create_viewTravaux.py
+68
View File
@@ -0,0 +1,68 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from pycen import con_fon as con
from sqlalchemy import text
db = 'bd_cen'
host = '91.134.194.221'
port = '5432'
srv_name = 'fdw_bd-cen-38'
tab_name = 'cr_line_saisie_travaux'
sch_name = 'travaux'
view_name = 'travaux_ligne'
user = 'cen_admin'
pwd = '#CEN38@venir'
dict_cols = {
'gid': 'serial4',
'geom': 'public.geometry(linestring, 2154)',
'id_site': 'varchar',
'annee_trav': 'int4',
'id_gestion': 'int4',
'id_presta': 'int4',
'etat_trav': 'varchar',
'nom_presta': 'varchar',
'date_debut': 'date',
'duree_trav': 'float8',
'nb_etp': 'float8',
'nb_balle': 'int4',
'val_mat': 'int4',
'legende': 'varchar',
'rmq': 'varchar',
'prestation': 'int4',
'long_m': 'float8',
'crea_date': 'date',
'date_update': 'date',
'date_fin': 'date',
}
col_table = ','.join([' '.join([key,val]) for key, val in dict_cols.items()])
table = '''
DROP FOREIGN TABLE IF EXISTS {sch_name}.{tab_name} CASCADE;
CREATE FOREIGN TABLE IF NOT EXISTS {sch_name}.{tab_name} (
{columns}
)
SERVER "{fgn_server}"
OPTIONS (schema_name '{sch_name}', table_name '{tab_name}');
'''.format(
sch_name=sch_name, tab_name=tab_name, columns=col_table, fgn_server=srv_name
)
col_view = ','.join(dict_cols.keys())
view = '''
CREATE OR REPLACE VIEW {sch_name}.{v_name} AS
SELECT
{columns}
FROM {sch_name}.{tab_name};
ALTER TABLE {sch_name}.{v_name} OWNER TO cen_admin;
GRANT ALL ON TABLE {sch_name}.{v_name} TO cen_admin;
GRANT SELECT ON TABLE {sch_name}.{v_name} TO grp_sig;
GRANT SELECT ON TABLE {sch_name}.{v_name} TO cen_user;
'''.format(
sch_name=sch_name, tab_name=tab_name, v_name=view_name, columns=col_view
)
with con.begin() as cnx:
cnx.execute(text(table))
cnx.execute(text(view))
0_FONCIER/TdB_FEDE/__init__.py
+31
View File
@@ -0,0 +1,31 @@
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
from os import getlogin
import pandas as pd
import geopandas as gpd
from sqlalchemy.engine import URL
from sqlalchemy import create_engine
from pycen import con_bdcen as con, con_fon
annee = 2023
path_source = '/media/{log}/SRV/FICHIERS/OUTILS/BASES DE DONNEES/BILAN_FEDE_CEN/{an}/Docs_SIG_joints{an}/Docs_SIG_joints{an}/'.format(log=getlogin(),an=annee)
dico_data = 'Dico_DATA_sites_CEN_v%i_v1.xlsx' % annee
sheet_par = 'parcelles_cen_XX_%i' % annee
sheet_sit = 'sites_cen_XX_%i' % annee
###### Récupération des données conservatoires
def get_sites_cen():
sqlsit = '''SELECT * FROM %s.%s WHERE date_fin is NULL OR date_fin >= '%i-01-01' ;'''%('sites','c_sites_zonages',annee)
dfsite = gpd.read_postgis(sql=sqlsit,geom_col='geom',con=con)
# Isolation de tous les sites CEN (gestion/assist/mili)
return dfsite[
~dfsite.code_site.duplicated()|\
(dfsite.code_site.duplicated() & dfsite.type_zonage == 'ZI')
]
def get_parcelles_cen():
return
0_FONCIER/TdB_FEDE/parcelles_TdB_fede .py
+732
View File
@@ -0,0 +1,732 @@
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
# library pandas geopandas sqlalchemy psycopg2-binary openpyxl
from calendar import c
from os import getlogin
import pandas as pd
import geopandas as gpd
from sqlalchemy.sql import text
from sqlalchemy.engine import URL
from sqlalchemy import create_engine
from pycen import con_bdcen as con, con_fon
from functools import reduce
annee = 2022
path_source = '/media/{login}/SRV/FICHIERS/OUTILS/BASES DE DONNEES/BILAN_FEDE_CEN/{an}/Docs_SIG_joints{an}/Docs_SIG_joints{an}/'.format(login=getlogin(),an=annee)
dico_data = 'Dico_DATA_sites_CEN_v%i_v1.xlsx' % annee
sheet_par = 'parcelles_cen_XX_%i' % annee
sheet_sit = 'sites_cen_XX_%i' % annee
lv_cen = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='A:D',nrows=25)
lv_typmaitr = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='F:K',nrows=23)
lv_typmilie = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='M:N',nrows=16)
lv_echelnum = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='P:Q',nrows=8)
lv_typhab = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=13, usecols='P:Q',nrows=5)
lv_sourcegeom = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='S:U',nrows=12)
lv_lienres = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=20, usecols='M:N',nrows=4)
lv_typprop = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=45, usecols='A:H',nrows=12)
lv_codgeol = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=36, usecols='M:T',nrows=19)
lst_cols_parc = pd.read_excel(path_source+dico_data,sheet_name=sheet_par,header=0, usecols='F',nrows=26)\
['nom du champ'].tolist()
user = 'postgres'
pwd = 'postgres'
adr = '172.17.0.2'
base = 'postgres'
url = URL.create('postgresql+psycopg2',
username=user,
password=pwd,
host=adr,
database=base,
)
con_vm = create_engine(url)
fadd = '/home/colas/Documents/tmp/FONCIER_FEDE/add_parc.gpkg'
add = gpd.read_file(fadd)
sql0 = '''SELECT
t1.geo_parcelle, substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) par_id,
t1.annee, t1.update_dat, t4.ccopre, t4.ccosec, t4.dcntpa, t1.geom, STRING_AGG(t3.gtoper::text,',') gtoper, STRING_AGG(t3.ccogrm::text,',') ccogrm,
STRING_AGG(CONCAT(t3.gtoper::text||COALESCE('_'||t3.ccogrm::text,'')),',') type_prop, STRING_AGG(t3.ddenom::text,',') ddenom
FROM "{sch}"."{t1}" t1
LEFT JOIN ("{sch}".parcelle_info t2
LEFT JOIN "{sch}".proprietaire t3 USING (comptecommunal))
USING (geo_parcelle)
LEFT JOIN "{sch}".parcelle t4 ON t1.geo_parcelle = t4.parcelle
'''.format(
sch='38_202207',
t1='v_geo_parcelle')
sql1 = '''WHERE substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) IN ('%s')
GROUP BY 1,2,3,4,5,6,7,8'''%"','".join(add.par_id)
sql = sql0 + sql1
parcx = gpd.read_postgis(text(sql),con=con_vm)
cols_parcx = parcx.columns
for col in ['ccopre','ccosec','type_prop','ddenom'] :
parcx[col] = parcx[col].str.strip()
parcx[col] = parcx[col].replace('',None)
parcx.loc[parcx.ccopre.isna(),'ccopre'] = parcx.loc[parcx.ccopre.isna(),'par_id'].str[5:8]
parcx.loc[parcx.ccosec.isna(),'ccosec'] = parcx.loc[parcx.ccosec.isna(),'par_id'].str[8:10]
parcx.loc[parcx.ccosec.str.len()==1,'ccosec'] = '0'+parcx.loc[parcx.ccosec.str.len()==1,'ccosec']
parcx.loc[parcx.dcntpa.isna(),'dcntpa'] = parcx.loc[parcx.dcntpa.isna(),'geom'].area.round().astype(int)
parcx.loc[parcx.gtoper.isna(),['gtoper','ccogrm','type_prop']] = ['2','0','2_0']
parcx.loc[parcx.par_id=='380740000B0705','ddenom'] = "CONSERVATOIRE D'ESPACES NATURELS ISERE"
parcx.dcntpa = parcx.dcntpa.astype(int)
###### Récupération des parcelles RN île de la Platière
f_plaRN = '/home/colas/Documents/tmp/FONCIER_FEDE/PLATIERE/Maitrise_Foncier_RN.shp'
f_plaENS = '/home/colas/Documents/tmp/FONCIER_FEDE/PLATIERE/Maitrise_Foncier_ENS.shp'
RNNP_rn = gpd.read_file(f_plaRN)
RNNP_ens = gpd.read_file(f_plaENS)
RNNP_rn.rename_geometry('geom',inplace=True)
RNNP_ens.rename_geometry('geom',inplace=True)
RNNP_rn.rename(columns={'Date_sign':'annee_acq','surf_acqui':'surface_ac','surf_parce':'superficie'},inplace=True)
RNNP_ens.rename(columns={'code_commu':'code_comm','année_acq':'annee_acq','année':'anne'},inplace=True)
RNNP_rn = RNNP_rn[RNNP_rn.code_comm.str.len()==5]
RNNP_ens = RNNP_ens[RNNP_ens.code_comm.str.len()==5]
RNNP_rn['section'] = RNNP_rn.parcelle.str[:2]
RNNP_ens['section'] = RNNP_ens.parcelle.str[:2]
RNNP_rn['num_parc'] = RNNP_rn.parcelle.str[2:]
RNNP_ens['num_parc'] = RNNP_ens.parcelle.str[2:]
RNNP_rn['num_parc'] = RNNP_rn['num_parc'].str.strip()
RNNP_ens['num_parc'] = RNNP_ens['num_parc'].str.strip()
RNNP_rn.loc[RNNP_rn.num_parc.str.len()==1,'num_parc'] = '000' + RNNP_rn.loc[RNNP_rn.num_parc.str.len()==1,'num_parc']
RNNP_rn.loc[RNNP_rn.num_parc.str.len()==2,'num_parc'] = '00' + RNNP_rn.loc[RNNP_rn.num_parc.str.len()==2,'num_parc']
RNNP_rn.loc[RNNP_rn.num_parc.str.len()==3,'num_parc'] = '0' + RNNP_rn.loc[RNNP_rn.num_parc.str.len()==3,'num_parc']
RNNP_ens.loc[RNNP_ens.num_parc.str.len()==1,'num_parc'] = '000' + RNNP_ens.loc[RNNP_ens.num_parc.str.len()==1,'num_parc']
RNNP_ens.loc[RNNP_ens.num_parc.str.len()==2,'num_parc'] = '00' + RNNP_ens.loc[RNNP_ens.num_parc.str.len()==2,'num_parc']
RNNP_ens.loc[RNNP_ens.num_parc.str.len()==3,'num_parc'] = '0' + RNNP_ens.loc[RNNP_ens.num_parc.str.len()==3,'num_parc']
RNNP_rn.loc[RNNP_rn.code_comm.str.startswith('39'),'code_comm'] = '39' + RNNP_rn.loc[RNNP_rn.code_comm.str.startswith('39'),'code_comm'].str[2:]
RNNP_ens.loc[RNNP_ens.code_comm.str.startswith('39'),'code_comm'] = '39' + RNNP_ens.loc[RNNP_ens.code_comm.str.startswith('39'),'code_comm'].str[2:]
RNNP_rn['par_id'] = RNNP_rn.code_comm+'000'+RNNP_rn.section+RNNP_rn.num_parc
RNNP_ens['par_id'] = RNNP_ens.code_comm+'000'+RNNP_ens.section+RNNP_ens.num_parc
lst_par_id_rnnp = [*RNNP_rn.par_id,*RNNP_ens.par_id]
RNNP = pd.concat([RNNP_rn,RNNP_ens])
RNNP['surf_parc_maitrise_m2'] = RNNP.surface_ac
RNNP['pour_part'] = 0
RNNP.loc[(~RNNP.proportion.isna())&(RNNP.proportion!=100),'pour_part'] = 1 # Récup après, cf. dfgere
frnnp_tofede = '/home/colas/Documents/tmp/FONCIER_FEDE/PLATIERE/parcelles_RNNP_tofede.gpkg'
rnnp_tofede = gpd.read_file(frnnp_tofede,layer='parcelles_RNNP_tofede')
rnnp_tofede.rename_geometry('geom', inplace=True)
rnnp_tofede['id_site_cen_parc'] = 0
rnnp_tofede['id_site_fcen_parc'] = None # OK
rnnp_tofede['insee_dep'] = rnnp_tofede.par_id.str[:2]
rnnp_tofede['insee_com'] = rnnp_tofede.par_id.str[2:5]
rnnp_tofede['num_section'] = rnnp_tofede.ccosec
rnnp_tofede['num_parcelle'] = rnnp_tofede.par_id
# rnnp_tofede['code_mfu1'] = None # Récup après, cf. dfgere
# rnnp_tofede['code_mfu2'] = None # Récup après, cf. dfgere
# rnnp_tofede['type_prop'] = None # Récup après, cf. parcb
# rnnp_tofede['mesure_compens'] = None # Récup après, cf. dfgere
rnnp_tofede['surf_ore_m2'] = 0
rnnp_tofede['date_debut_ore'] = None
rnnp_tofede['date_fin_ore'] = None
rnnp_tofede['doc_foncier'] = 0
rnnp_tofede['source_doc_foncier'] = 0
rnnp_tofede['parc_gestion_rnx'] = 0
rnnp_tofede['parc_gestion_rnx'] = 1
rnnp_tofede['surf_parc_maitrise_m2'] = rnnp_tofede.dcntpa
rnnp_tofede['source_geom_parc_nature'] = 2
rnnp_tofede['source_geom_parc_annee'] = None # Récup après, cf. parcb
rnnp_tofede['echelle_num_parc'] = 0
rnnp_tofede['source_surf_parc'] = 1 # OK
rnnp_tofede['date_maj_parcelle'] = None # Récup après, cf. parcb
rnnp_tofede['bnd'] = None # Récup après, cf. dfgere
rnnp_tofede['pour_part'] = None # Récup après, cf. dfgere
rnnp_tofede['domaine_public'] = 0
rnnp_tofede['id_proprietaire'] = None # Récup après, cf. parcb
# RNNP_ens[(~RNNP_ens.num_parcel.isna())&(RNNP_ens.num_parcel!=RNNP_ens.parcelle)]
###### Récupération des données conservatoires
sqlsit = '''SELECT * FROM %s.%s WHERE date_fin is NULL OR date_fin >= '%i-01-01' ;'''%('sites','c_sites_zonages',annee)
dfsite = gpd.read_postgis(sql=sqlsit,geom_col='geom',con=con)
# Isolation de tous les sites CEN (gestion/assist/mili)
dfsite[
~dfsite.code_site.duplicated()|\
(dfsite.code_site.duplicated() & dfsite.type_zonage == 'ZI')
]
rngl = dfsite[(dfsite.code_site=='RNGL')&(dfsite.type_zonage=='ZI')]
sqlfon = 'SELECT * FROM %s.%s'%('foncier','c_animation_fonciere_sites')
dffonc = gpd.read_postgis(sql=sqlfon,geom_col='geom',con=con)
##### Données identifiées pour les remontées fédé
ffede = '/home/colas/Documents/tmp/FONCIER_FEDE/extract_parc_for_fede.gpkg'
tofede = gpd.read_file(ffede)
tofede.rename_geometry('geom', inplace=True)
cols_tofede = tofede.columns
cols_join = [*cols_parcx[cols_parcx.isin(cols_tofede)]]
tofede = pd.concat([tofede,parcx[cols_join]],ignore_index=True)
tofede.reset_index(drop=True,inplace=True)
if 'type_prop' in tofede.columns:
tofede.drop(columns=['type_prop'], inplace=True)
# tofede['id_site_cen_parc'] = 0
tofede['id_site_fcen_parc'] = None # OK
tofede['insee_dep'] = tofede.par_id.str[:2]
tofede['insee_com'] = tofede.par_id.str[2:5]
tofede['num_section'] = tofede.ccosec
tofede['num_parcelle'] = tofede.par_id
# tofede['code_mfu1'] = None # Récup après, cf. dfgere
# tofede['code_mfu2'] = None # Récup après, cf. dfgere
# tofede['type_prop'] = None # Récup après, cf. parcb
# tofede['mesure_compens'] = None # Récup après, cf. dfgere
tofede['surf_ore_m2'] = 0
tofede['date_debut_ore'] = None
tofede['date_fin_ore'] = None
tofede['doc_foncier'] = 0
tofede['source_doc_foncier'] = 0
tofede['parc_gestion_rnx'] = 0
tofede.loc[tofede.geom.intersects(rngl.unary_union),'parc_gestion_rnx'] = 1
tofede['surf_parc_maitrise_m2'] = tofede.dcntpa
tofede['source_geom_parc_nature'] = 2
# tofede['source_geom_parc_annee'] = None # Récup après, cf. parcc
tofede['echelle_num_parc'] = 0
tofede['source_surf_parc'] = 1 # OK
# tofede['date_maj_parcelle'] = None # Récup après, cf. parcc
# tofede['bnd'] = None # Récup après, cf. dfgere
# tofede['pour_part'] = None # Récup après, cf. dfgere
tofede['domaine_public'] = 0
# tofede['id_proprietaire'] = None # Récup après, cf. parcb
# c_fede = tofede[['par_id','geom']].copy()
# c_fede.geom = tofede.representative_point()
# pb_parcel = dffonc[~dffonc.intersects(c_fede.unary_union)]
# c_fede.to_file('/home/colas/Documents/tmp/FONCIER_FEDE/centroid_parcelle_for_fede.gpkg', driver="GPKG")
# pb_parcel.to_file('/home/colas/Documents/tmp/FONCIER_FEDE/polys_NoIntersect_parcelles.gpkg', driver="GPKG")
###### Récupération des données conservatoires
# sqlsit = 'SELECT * FROM %s.%s'%('sites','c_sites_zonages')
# dfsite = gpd.read_postgis(sql=sqlsit,geom_col='geom',con=con)
sqlfon = 'SELECT * FROM %s.%s'%('foncier','c_animation_fonciere_sites')
dffonc = gpd.read_postgis(sql=sqlfon,geom_col='geom',con=con)
# dffonc.drop(columns=[], inplace=True)
#### 1 geom devient 2
# id 7037
dffonc.loc[dffonc.id==7037, ['code_dep','code_com','section','numero','geom']] = tofede.loc[tofede.par_id=='381000000A2233',['insee_dep','insee_com','num_section','dnupla','geom']].values[0]
add_p = gpd.GeoDataFrame(
[tofede.loc[tofede.par_id=='381000000A2234',['insee_dep','insee_com','num_section','dnupla','geom']].values[0]],
columns=['code_dep','code_com','section','numero','geom']
)
oth_cols = dffonc.columns[~dffonc.columns.isin(add_p.columns)].drop('id')
add_p[[*oth_cols]] = dffonc.loc[dffonc.id==7037, [*oth_cols]].values[0]
add_p['id'] = 0
add_p.set_geometry('geom', crs=2154, inplace=True)
dffonc = pd.concat([dffonc,add_p])
# id 1932
dffonc.loc[dffonc.id==1932, ['code_dep','code_com','section','numero','geom']] = tofede.loc[tofede.par_id=='38140000AM0190',['insee_dep','insee_com','num_section','dnupla','geom']].values[0]
add_p = gpd.GeoDataFrame(
[tofede.loc[tofede.par_id=='38140000AM0189',['insee_dep','insee_com','num_section','dnupla','geom']].values[0]],
columns=['code_dep','code_com','section','numero','geom']
)
oth_cols = dffonc.columns[~dffonc.columns.isin(add_p.columns)].drop('id')
add_p[[*oth_cols]] = dffonc.loc[dffonc.id==1932, [*oth_cols]].values[0]
add_p['id'] = 0
add_p.set_geometry('geom', crs=2154, inplace=True)
dffonc = pd.concat([dffonc,add_p])
####
dropline = dffonc[dffonc.id.isin([7144,7106,7107,7108,7109,7110,6666])].index.values # drop geometrie dupliquée
dffonc.drop([*dropline], inplace=True)
dffonc.loc[dffonc.numero.str.len()==1,'numero'] = '000'+dffonc.loc[dffonc.numero.str.len()==1,'numero']
dffonc.loc[dffonc.numero.str.len()==2,'numero'] = '00'+dffonc.loc[dffonc.numero.str.len()==2,'numero']
dffonc.loc[dffonc.numero.str.len()==3,'numero'] = '0'+dffonc.loc[dffonc.numero.str.len()==3,'numero']
dffonc.loc[dffonc.section.str.len()==1,'section'] = '0'+dffonc.loc[dffonc.section.str.len()==1,'section']
dffonc.loc[dffonc.id == 7101,['code_dep','code_com','section','numero']] = ['38','037','0D','0130']
dffonc.loc[dffonc.id == 7102,['code_dep','code_com','section','numero']] = ['38','037','0D','0131']
dffonc.loc[dffonc.id == 7103,['code_dep','code_com','section','numero']] = ['38','037','0D','0132']
dffonc.loc[dffonc.id == 7104,['code_dep','code_com','section','numero']] = ['38','037','0D','0133']
dffonc.loc[dffonc.id == 7105,['code_dep','code_com','section','numero']] = ['38','037','0D','0134']
dffonc.loc[dffonc.id.isin([
61,62,70,69,68,72,67,66,63,64,60,57,56,59,6,7,8,9,65,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,
26,27,28,29,30,58,31,32,33,34,35,36,37,38,39,40,41,55,54,42,43,71,44,45,46,47,48,49,50,51,52,53,4,5,
1350,1349,1345,1346,1347,1352,1353,1348,1354,1351, # ccopre == 121
2433,2447,2449,2432,2448 # ccopre == 274
]),'code_com'] = '479'
dffonc.loc[dffonc.id.isin([
2604,2605,2535,2599,2603,2602,2601,2639,2534,2536,2537,2532,2538,2540,2539,2635,2641,2533,2563,2562,
2561,2556,2560,2555,2597,2607,2614,2591,2622,2623,2620,2621,2619,2593,2592,2550,2547,2544,2551,2553,
2552,2545,2541,2546,2548,2549,2543,2583,2566,2596,2608,2567,2559,2584,2568,2615,2629,2618,2631,2571,
2609,2572,2590,2632,2569,2570,2595,2558,2636,2633,2606,2574,2573,2564,2579,2580,2577,2578,2565,2557,
2576,2581,2613,2598,2626,2627,2625,2611,2624,2642,2554,2628,2582,2585,2610,2542,2594,2589,2575,2616,
2612,2600,2617,2630,2587,2588,2586,2638,2637,2640,2634
]),'code_com'] = '292'
dffonc.loc[dffonc.id.isin([7114,7115,7116,7117,7118,7119,7126,7127,7128,7129,7130,7131,7132,7133,7134,7135]),'code_com'] = '064'
dffonc.loc[dffonc.id.isin([7120,7121,7122,7123,7124,7125]),'code_com'] = '401'
dffonc.loc[dffonc.id.isin([7136,7137,7138,7139]),['code_com','section']] = ['341','0B']
dffonc.loc[dffonc.id.isin([7142,7143]),'code_com'] = '052'
dffonc.loc[dffonc.id.isin([1010288,1041363]),'code_com'] = '517'
dffonc.loc[dffonc.id.isin([1182503,1182640]),'code_com'] = '147'
dffonc.loc[dffonc.id.isin([1243657]),'code_com'] = '448'
dffonc.loc[dffonc.id.isin([1629034,1629035]),'code_com'] = '257'
dffonc.loc[dffonc.code_dep.isna(),'code_dep'] = '38'
#####
dffonc.loc[dffonc.id==113, 'numero'] = '0389'
dffonc.drop(dffonc[dffonc.id==114].index[0],inplace=True)
dffonc.loc[dffonc.id==151, 'numero'] = '0369'
dffonc.loc[dffonc.id==591, 'numero'] = '0705' # numero == '0542' est un ancien nom
dffonc.sort_values(['code_dep','code_com','section','numero'],inplace=True)
dffonc['par_id'] = dffonc.code_dep + dffonc.code_com + '___' + dffonc.section + dffonc.numero
dffonc['parid_start'] = dffonc.code_dep + dffonc.code_com
dffonc['parid_end'] = dffonc.section + dffonc.numero
dffonc.loc[dffonc.id.isin([1634,1638,1606,1607,1608]),'par_id'] = dffonc.loc[dffonc.id.isin([1634,1638,1606,1607,1608]),'par_id'].replace('___','227',regex=True)
dffonc.loc[dffonc.id.isin([1495,1673,1675]),'par_id'] = dffonc.loc[dffonc.id.isin([1495,1673,1675]),'par_id'].replace('___','000',regex=True)
# dffonc.loc[dffonc.id.isin([1495,1675]),'par_id']
# Jointure by parid_start & parid_end
dfpar_id = tofede[['par_id']].copy()
dfpar_id['parid_start'] = dfpar_id.par_id.str[:5]
dfpar_id['parid_end'] = dfpar_id.par_id.str[-6:]
lst_parid_ok = [1495,1634,1638,1673,1675,1606,1607,1608] # liste parcelle par_id reconstitué entier
parid_ok = dffonc.loc[dffonc.id.isin(lst_parid_ok)]
parid_join = pd.merge(
dffonc.loc[~dffonc.id.isin(lst_parid_ok)].drop(columns=['par_id']),
dfpar_id.loc[~dfpar_id.par_id.isin(parid_ok.par_id)],
on = ['parid_start','parid_end'],
how='left'
)
parid_join.geom = parid_join.buffer(0)
parid_ok = pd.concat([ parid_ok, parid_join[~parid_join.par_id.isna()] ])
parid_ok.drop(columns=['parid_start','parid_end'], inplace=True)
parid_na = parid_join[parid_join.par_id.isna()]
# Jointure by centroid
dfpar_id = tofede.loc[~tofede.par_id.isin(parid_ok.par_id),['par_id','geom']].copy()
dfpar_id.geom = dfpar_id.representative_point()
parid_join = gpd.sjoin(parid_na.drop(columns=['par_id']),dfpar_id,op='contains', how='left').drop(columns=['index_right'])
parid_join_ok = parid_join[ [str(x).startswith(y) and z in str(x) for x,y,z in zip(parid_join['par_id'], parid_join['parid_start'],parid_join['section'])] ].copy()
# parid_join[ [str(x).endswith(z) for x,y,z in zip(parid_join['par_id'], parid_join['parid_start'],parid_join['parid_end'])] ]
parid_join_ok.drop(columns=['parid_start','parid_end'], inplace=True)
# Couches annimation foncière finale
parid_OK = pd.concat([parid_ok,parid_join_ok])
parid_NA = dffonc[~dffonc.id.isin(parid_OK.id.unique())]
# Récupération des info BND
sqlmod = 'SELECT * FROM %s'%('animation_fonciere_pour_fcen')
dfmodf = gpd.read_postgis(sql=sqlmod,geom_col='geom',con=con)
del dfmodf['id_0']
dffon_OK = pd.merge(
parid_OK,
dfmodf[['id','Parc_MCE','docs_off_num','Nbr_lots_BND','partiel_conv_ou_acq']],
on='id',how='left')
#
lst_prop = ['Acquis par le CEN ou CREN','ACQUIS&CONVENTION', 'PARTIELLEMENT MAITRISE']
lst_conv = ["CONVENTION D'USAGE",'CONVENTION']
lst_gere = ['BAIL EMPHYTEOTIQUE', *lst_conv, *lst_prop]
dfgere = dffon_OK[
# (
# (dffon_OK.geom.intersects(dfsite[(dfsite.type_zonage=='ZI')|(dfsite.type_site=='MC')].unary_union))|
# (~dffon_OK.Parc_MCE.isna())|(~dffon_OK.Nbr_lots_BND.isna())|
# (~dffon_OK.partiel_conv_ou_acq.isna())|(~dffon_OK.docs_off_num.isna())
# )
# &
(dffon_OK.maitrise.isin(lst_gere))
].copy()
dfautr = dffon_OK[~dffon_OK.par_id.isin(dfgere.par_id)].copy()
# id_site_cen_parc
dfgere['id_site_cen_parc'] = dfgere.id
# code_mfu1 & code_mfu2
dfgere['code_mfu1'] = None
dfgere['code_mfu2'] = None
dfgere.loc[dfgere.maitrise.isin(lst_prop),'code_mfu1'] = 'P'
dfgere.loc[dfgere.maitrise.isin(lst_conv),'code_mfu1'] = 'C'
dfgere.loc[dfgere.maitrise == 'BAIL EMPHYTEOTIQUE','code_mfu1'] = 'L'
dfgere.loc[dfgere.maitrise.isin(lst_prop),'code_mfu2'] = 'P1'
dfgere.loc[dfgere.maitrise.isin(lst_conv),'code_mfu2'] = 'C7'
dfgere.loc[dfgere.maitrise == 'BAIL EMPHYTEOTIQUE','code_mfu2'] = 'L1'
# BND
dfgere['bnd'] = 0
dfgere.loc[dfgere.indivision=='BND','bnd'] = 1
tmp = pd.DataFrame(columns=dfgere.columns)
for i,row in dfgere.loc[~dfgere.Nbr_lots_BND.isna()].iterrows():
# if i==0 : tmp = row.to_list()
tmp = tmp.append([row]*(int(row.Nbr_lots_BND)-1))
dfgere = pd.concat([dfgere,tmp]).sort_index()
# MC
dfgere['mesure_compens'] = 0
dfgere.loc[dfgere.Parc_MCE=='OUI','mesure_compens'] = 1
# pour_part
dfgere['pour_part'] = 0
dfgere.loc[dfgere.maitrise=='PARTIELLEMENT MAITRISE','pour_part'] = 1
# Parcelles millitaire
dic = {
'ccosec':'section',
}
dfmili = dfsite[dfsite.type_site=='MILI'].copy()
sqlmil = "SELECT * FROM %s.%s WHERE ST_Contains('SRID=2154;%s',geom)" % ('cadastre', 'parcelles_38',dfmili.buffer(10).unary_union)
miliparc = gpd.read_postgis(sqlmil,con=con_fon)
miliparc['code_dep'] = miliparc.codcom.str[:2]
miliparc['code_com'] = miliparc.codcom.str[2:]
dfmili = miliparc[['par_id','geom','ccosec','code_dep','code_com']].copy()
dfmili.rename(columns=dic, inplace=True)
dfmili['id_site_cen_parc'] = 0
dfmili['code_mfu1'] = 'C'
dfmili['code_mfu2'] = 'C12'
dfmili['bnd'] = 0
dfmili['mesure_compens'] = 0
dfmili['pour_part'] = 0
dfgere = pd.concat([dfgere,dfmili])
# from pycen import update_to_sql
# tmpdffonc = dffonc[['id','nom_propri']]
# tmpdffonc["nom_propri"] = tmpdffonc["nom_propri"].str.strip()
# tmpdffonc.loc[tmpdffonc.nom_propri== '',"nom_propri"] = None
# update_to_sql(df=tmpdffonc, con=con, table_name='c_animation_fonciere_sites', schema_name='foncier', key_name='id', geom_col=None)
#####################
##### Parcelles #####2017380474000AN
# Ne considère pas Platière, ni le Grand Lemps
#####################
# Récupération des infos dans la bdd bd_cen (#Foncier)
# RNNP
sql = '''SELECT DISTINCT ON (t1.par_id,t1.geom, t1.dcntpa,t1.typprop_id,t1.ccopre,t1.ccosec,t1.dnupla)
t1.par_id,t1.geom, t1.dcntpa,t1.typprop_id,t1.ccopre,
t1.ccosec,t1.dnupla::text, STRING_AGG(t6.gtoper::text,',') gtoper, STRING_AGG(t6.ccogrm::text,',') ccogrm,
STRING_AGG(CONCAT(t6.gtoper::text||COALESCE('_'||t6.ccogrm::text,'')),',') type_prop, STRING_AGG(t6.ddenom::text,',') ddenom
FROM {sch}."{t1}" t1
JOIN ({sch}."lots_38" t2
JOIN ({sch}."cadastre_38" t3
JOIN ({sch}."cptprop_38" t4
JOIN ({sch}."r_prop_cptprop_38" t5
JOIN {sch}."proprios_38" t6 USING (dnuper))
USING (dnupro))
USING (dnupro))
USING (lot_id))
USING (par_id)
WHERE par_id IN ('{par_ids}') GROUP BY 1,2,3,4,5,6'''.format(
sch='cadastre',
t1='parcelles_38',
par_ids="','".join(lst_par_id_rnnp))
parca1 = gpd.read_postgis(text(sql),con=con_fon)
parca1.to_file('/home/colas/Documents/tmp/FONCIER_FEDE/PLATIERE/parcelles_RNNP_tofede.gpkg', driver='GPKG')
# Récupération des infos dans la bdd bd_cen (#Foncier)
sql = '''SELECT DISTINCT ON (t1.par_id,t1.geom, t1.dcntpa,t1.typprop_id,t1.ccopre,t1.ccosec,t1.dnupla)
t1.par_id,t1.geom, t1.dcntpa,t1.typprop_id,t1.ccopre,
t1.ccosec,t1.dnupla::text, STRING_AGG(t6.gtoper::text,',') gtoper, STRING_AGG(t6.ccogrm::text,',') ccogrm,
STRING_AGG(CONCAT(t6.gtoper::text||COALESCE('_'||t6.ccogrm::text,'')),',') type_prop, STRING_AGG(t6.ddenom::text,',') ddenom
FROM {sch}."{t1}" t1
JOIN ({sch}."lots_38" t2
JOIN ({sch}."cadastre_38" t3
JOIN ({sch}."cptprop_38" t4
JOIN ({sch}."r_prop_cptprop_38" t5
JOIN {sch}."proprios_38" t6 USING (dnuper))
USING (dnupro))
USING (dnupro))
USING (lot_id))
USING (par_id)
WHERE par_id IN ('{par_ids}') GROUP BY 1,2,3,4,5,6'''.format(
sch='cadastre',
t1='parcelles_38',
par_ids="','".join(tofede.par_id))
parca2 = gpd.read_postgis(text(sql),con=con_fon)
cols_parca2 = parca2.columns
cols_join = [*cols_parcx[cols_parcx.isin(cols_parca2)]]
parcb = pd.concat([
parca1,
parca2[~parca2.par_id.isin(parcx.par_id)],
parcx[cols_join]
])
# n = 10
# while n!= 0:
# parcb.type_prop = parcb.type_prop.replace(',,',',',regex=True)
# n = n-1
parcb.loc[parcb.type_prop.str.contains('2_0'),['type_prop','gtoper','ccogrm']] = ['2_0_MIX','2','0']
parcb.loc[parcb.type_prop.isin(['2_7,2_3,','2_7,2_3,2_3,']),['type_prop','gtoper','ccogrm']] = ['2_0_MIX','2','0']
parcb.loc[parcb.type_prop.str.contains('2_9'),['type_prop','gtoper','ccogrm']] = ['2_9_PUB','2','9']
parcb.loc[parcb.type_prop.str.contains('2_8'),['type_prop','gtoper','ccogrm']] = ['2_8_PRI','2','8']
parcb.loc[parcb.type_prop.str.contains('2_7'),['type_prop','gtoper','ccogrm']] = ['2_7_PRI','2','7']
parcb.loc[parcb.type_prop.str.contains('2_6'),['type_prop','gtoper','ccogrm']] = ['2_6_PRI','2','6']
parcb.loc[parcb.type_prop.str.contains('2_5'),['type_prop','gtoper','ccogrm']] = ['2_5_PUB','2','5']
parcb.loc[parcb.type_prop.str.contains('2_4'),['type_prop','gtoper','ccogrm']] = ['2_4_COM','2','4']
parcb.loc[parcb.type_prop.str.contains('2_3'),['type_prop','gtoper','ccogrm']] = ['2_3_PUB','2','3']
parcb.loc[parcb.type_prop.str.contains('2_2'),['type_prop','gtoper','ccogrm']] = ['2_2_PUB','2','2']
parcb.loc[parcb.type_prop.str.contains('2_1'),['type_prop','gtoper','ccogrm']] = ['2_1_PUB','2','1']
parcb.loc[parcb.type_prop.str.contains('1,1'),['type_prop','gtoper','ccogrm']] = ['1_PRI','1',None]
parcb.loc[parcb.type_prop == '1',['type_prop','gtoper','ccogrm']] = ['1_PRI','1',None]
lstterm_cen38 = ["CONSERVATOIRE D'ESPACES NATURELS ISERE","AVENIR CONSERVATOIRE DES ESPACES NATURELS DE L'ISERE","CONSERVATOIRE D'ESPACES NATURELS ISERE AVENIR DIT CEN ISERE"]
lstterm_cenra = ["CONSERVATOIRE RHONE ALPES DES ESPACES NATURELS","CONSERVATOIRE RH ALP ESPACES NATURELS"]
parcb.loc[parcb.ddenom.isin([*lstterm_cen38,*lstterm_cenra]),['type_prop','gtoper','ccogrm']] = ['3_CEN','3',None]
# parcb.loc[parcb.type_prop.str.len()==3,'gtoper'] = parcb[parcb.type_prop.str.len()==3].type_prop.str[0]
# parcb.loc[parcb.type_prop.str.len()==3,'ccogrm'] = parcb[parcb.type_prop.str.len()==3].type_prop.str[-1]
parcb.loc[parcb.dnupla.str.len()==1,'dnupla'] = '000' + parcb.loc[parcb.dnupla.str.len()==1,'dnupla']
parcb.loc[parcb.dnupla.str.len()==2,'dnupla'] = '00' + parcb.loc[parcb.dnupla.str.len()==2,'dnupla']
parcb.loc[parcb.dnupla.str.len()==3,'dnupla'] = '0' + parcb.loc[parcb.dnupla.str.len()==3,'dnupla']
parcb['id_proprietaire'] = 0
parcb.loc[parcb.ddenom.str.contains('|'.join(lstterm_cenra),na=False),'id_proprietaire'] = '28'
# Récupération des infos dans la VM cadastre (#Foncier)
# sql = '''SELECT geo_parcelle, substring(geo_parcelle from 1 for 2)||substring(geo_parcelle from 4 for 12) par_id, geom
# FROM "38_202207".geo_parcelle'''
# 38517000AW0032 # par_id
# 380517000AW0032 # geo_parcelle
sql = '''SELECT
substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) par_id,
t1.annee, t1.update_dat, t1.geom
FROM "{sch}"."{t1}" t1
WHERE substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) IN ('{par_ids}')
'''.format(
sch='38_202207',
t1='v_geo_parcelle',
par_ids="','".join(tofede[~tofede.par_id.isin(parcx.par_id)].par_id))
parcc0 = gpd.read_postgis(text(sql),con=con_vm)
sql = '''SELECT
substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) par_id,
t1.annee, t1.update_dat, t1.geom
FROM "{sch}"."{t1}" t1
WHERE substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) IN ('{par_ids}')
'''.format(
sch='26_202007',
t1='v_geo_parcelle',
par_ids="','".join(tofede[~tofede.par_id.isin(parcx.par_id)].par_id))
parcc1 = gpd.read_postgis(text(sql),con=con_vm)
cols_parcc = parcc0.columns
cols_join = [*cols_parcx[cols_parcx.isin(cols_parcc)]]
parcc = pd.concat([parcc0,parcc1,parcx[cols_join]])
parcc['source_geom_parc_annee'] = parcc.annee
parcc['date_maj_parcelle'] = parcc.update_dat
l1 = dfgere.columns[dfgere.columns.isin(lst_cols_parc)]
l2 = tofede.columns[tofede.columns.isin(lst_cols_parc)]
l3 = parcb.columns[parcb.columns.isin(lst_cols_parc)]
l4 = parcc.columns[parcc.columns.isin(lst_cols_parc)]
p1 = dfgere[['par_id',*l1]].copy()
p2 = tofede[['par_id',*l2,'geom']].copy()
p3 = parcb[['par_id',*l3]].copy()
p4 = parcc[['par_id',*l4]].copy()
dfs = [p1,p2,p3,p4]
df_parcs = reduce(lambda left,right: pd.merge(left,right,on=['par_id'],
how='left'), dfs)
for par_id in df_parcs[df_parcs.par_id.duplicated(keep=False)].par_id.unique():
x = df_parcs[df_parcs.par_id==par_id].shape[0]
df_parcs.loc[df_parcs.par_id==par_id,'surf_parc_maitrise_m2'] = round(df_parcs.loc[df_parcs.par_id==par_id,'surf_parc_maitrise_m2']/x).astype(int)
df_parcs.date_maj_parcelle = df_parcs.date_maj_parcelle.astype(str)
gdf_tofede = df_parcs[['id_site_fcen_parc','num_parcelle','date_maj_parcelle','geom']].copy()
gdf_tofede = gdf_tofede.set_geometry('geom', crs=2154)
gdf_tofede.rename(columns={
'id_site_fcen_parc':'id_fcen',
'num_parcelle':'num_parc',
'date_maj_parcelle':'date_maj_p'
},inplace=True)
gdf_tofede.to_file('/home/colas/Documents/tmp/FONCIER_FEDE/SEND/Parcelles_CEN_38_%i.shp' % annee)
df_tofede = df_parcs[lst_cols_parc].copy()
df_tofede.loc[df_tofede.date_debut_ore.isna(),'date_debut_ore']
df_tofede.to_csv('/home/colas/Documents/tmp/FONCIER_FEDE/SEND/Parcelles_CEN_38_%i.csv' % annee)
# parcb.loc[(parcb.ddenom.str.contains("AVENIR CONSERVATOIRE DES ESPACES NATURELS DE L'ISERE",case=False))].ddenom.unique()
# parcb.loc[(parcb.ddenom.str.contains('CONSERVATOIRE|CEN',case=False))&(parcb.ddenom.str.contains('ISERE',case=False))].ddenom.unique()
# parcb.loc[parcb.gtoper.isin(['1,1','2,1','1,2','1,1,1']),'gtoper'] = '1'
# parcb.loc[parcb.gtoper.isin(['1,1,1,1','2,1,1,1,1,1,1','2,1,1,1,1,1,1,1,1,1,1,1,1,1,1','2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1']),'gtoper'] = '1'
# parcb.loc[parcb.gtoper.isin(['2,2']),'gtoper'] = '2'
# parcb.loc[parcb.gtoper=='1','ccogrm'] = None
parcb[parcb.par_id.duplicated(keep=False)]
dic_col = {
'code_dep' : 'insee_dep',
'section' : 'num_section',
'num_unique_v2' : 'num_parcelle', # ????
}
dic_parc = {
'id':'id_site_cen_parc',
}
lst_prop = ['Acquis par le CEN ou CREN','ACQUIS&CONVENTION', 'PARTIELLEMENT MAITRISE']
lst_conv = ["CONVENTION D'USAGE",'CONVENTION']
lst_gere = ['BAIL EMPHYTEOTIQUE', *lst_conv, *lst_prop]
sql = '''SELECT DISTINCT
t1.par_id,t1.geom, t1.dcntpa,t1.typprop_id,t1.ccopre,
t1.ccosec,t1.dnupla::text, t6.gtoper::text, t6.ccogrm::text
FROM {sch}."{t1}" t1
JOIN ({sch}."lots_38" t2
JOIN ({sch}."cadastre_38" t3
JOIN ({sch}."cptprop_38" t4
JOIN ({sch}."r_prop_cptprop_38" t5
JOIN {sch}."proprios_38" t6 USING (dnuper))
USING (dnupro))
USING (dnupro))
USING (lot_id))
USING (par_id)
WHERE ST_Within(t1.geom, 'SRID=2154;{geom}') '''.format(
sch='cadastre',
t1='parcelles_38',
geom=dffonc.unary_union)
sql = '''SELECT DISTINCT
t1.par_id,t1.geom, t1.dcntpa,t1.typprop_id,t1.ccopre,
t1.ccosec,t1.dnupla::text, t6.gtoper::text, t6.ccogrm::text
FROM {sch}."{t1}" t1
JOIN ({sch}."lots_38" t2
JOIN ({sch}."cadastre_38" t3
JOIN ({sch}."cptprop_38" t4
JOIN ({sch}."r_prop_cptprop_38" t5
JOIN {sch}."proprios_38" t6 USING (dnuper))
USING (dnupro))
USING (dnupro))
USING (lot_id))
USING (par_id)
WHERE ST_Intersects(t1.geom, 'SRID=2154;{geom}') '''.format(
sch='cadastre',
t1='parcelles_38',
geom=dffonc.unary_union)
sql = sql + " AND t1.par_id LIKE ANY (array['%s']);"% "','".join(dffonc[~dffonc.par_id.isna()].par_id)
parc = gpd.read_postgis(sql=text(sql),geom_col='geom',con=con_fon)
parc.loc[parc.dnupla.str.len()==1,'dnupla'] = '000' + parc.loc[parc.dnupla.str.len()==1,'dnupla']
parc.loc[parc.dnupla.str.len()==2,'dnupla'] = '00' + parc.loc[parc.dnupla.str.len()==2,'dnupla']
parc.loc[parc.dnupla.str.len()==3,'dnupla'] = '0' + parc.loc[parc.dnupla.str.len()==3,'dnupla']
# parc = gpd.read_file('/home/colas/Documents/tmp/FONCIER_FEDE/extract_parc_for_fede.gpkg')
# parc.rename_geometry('geom',inplace=True)
# parc.to_file('/home/colas/Documents/tmp/FONCIER_FEDE/extract_parc_for_fede.gpkg',driver='GPKG')
parc['semi_par_id'] = parc.par_id.str[:5] + parc.ccosec + parc.dnupla
######################
# HARMONISATION couche FONCIER
dffonc_b1 = dffonc_b5 = dffonc_b20 = dffonc_b50 = dffonc.copy()
dffonc_b1.geom = dffonc_b1.buffer(1)
dffonc_b20.geom = dffonc_b20.buffer(20)
dffonc_b50.geom = dffonc_b50.buffer(50)
dffonc_b5.geom = dffonc_b5.buffer(5)
tmp = gpd.sjoin(dffonc_b50[~dffonc_b50.numero.isna()],parc,op='contains',how='left')
tmp2 = tmp[tmp.numero == tmp.dnupla]
crsp = tmp2.id.tolist()
tmp[~tmp.id.isin(crsp)]
tmp[tmp.id.duplicated(keep=False)]
##############
##############
dfparc = parc[['par_id','gtoper','ccogrm','geom']]
dfgere = dffonc[dffonc.maitrise.isin(lst_gere)].copy() # parcelles gérée
dfgere['code_mfu1'] = None
dfgere['code_mfu2'] = None
dfgere.loc[dfgere.maitrise.isin(lst_prop),'code_mfu1'] = 'P'
dfgere.loc[dfgere.maitrise.isin(lst_conv),'code_mfu1'] = 'C'
dfgere.loc[dfgere.maitrise == 'BAIL EMPHYTEOTIQUE','code_mfu1'] = 'L'
dfgere.loc[dfgere.maitrise.isin(lst_prop),'code_mfu2'] = 'P1'
dfgere.loc[dfgere.maitrise.isin(lst_conv),'code_mfu2'] = 'C7'
dfgere.loc[dfgere.maitrise == 'BAIL EMPHYTEOTIQUE','code_mfu2'] = 'L1'
dfgere.loc[dfgere.maitrise == 'PARTIELLEMENT MAITRISE','bnd'] = 1
dfgere.loc[dfgere.maitrise != 'PARTIELLEMENT MAITRISE','bnd'] = 0
# Parcelles d'interventions
dftmp = dffonc[~dffonc.maitrise.isin(lst_gere)].copy() # couche temp des parcelles non-gérés
sittmp = dfsite[(dfsite.assist_cnv)&(dfsite.assist_col)] # sites en assistance
intersect = dftmp.intersects(sittmp.unary_union) # Intersection [couche temp] VS [sites assist]
dfintr = dftmp[intersect] # parcelles d'interventions
# Parcelles millitaire
dic = {
'ccosec':'num_section',
'par_id':'num_parcelle'
}
dfmili = dfsite[dfsite.type_site=='MILI'].copy()
sqlmil = "SELECT * FROM %s.%s WHERE ST_Contains('SRID=2154;%s',geom)" % ('cadastre', 'parcelles_38',dfmili.buffer(10).unary_union)
miliparc = gpd.read_postgis(sqlmil,con=con_fon)
miliparc['code_dep'] = miliparc.codcom.str[:2]
miliparc['code_com'] = miliparc.codcom.str[2:]
dfmili = miliparc[['par_id','geom','ccosec','code_dep','code_com']].copy()
dfmili.rename(columns=dic, inplace=True)
# All parcelles
df = pd.concat([dfgere,dfintr,dfmili], ignore_index=True)
keep_col = df.columns[df.columns.isin(lst_cols_parc)]
df = df[keep_col]
df.reset_index(drop=True, inplace=True)
sql = '''SELECT par_id,geom, dcbtpa,typprop_id FROM %s."%s" WHERE ST_Intersects(geom, 'SRID=2154;%s') ''' % ('cadastre', 'parcelles_38',df.representative_point().unary_union)
parc = gpd.read_postgis(sql=sql,geom_col='geom',con=con_fon)
sql = '''SELECT geo_parcelle,geo_section,idu,proprietaire,proprietaire_info,geom FROM %s."%s" WHERE ST_Intersects(geom, 'SRID=2154;%s') ''' % ('ref_cadastre', 'parcelle_info',df.representative_point().unary_union)
prop = gpd.read_postgis(sql=sql,geom_col='geom',con=con)
df['insee_com'] = df.code_dep + df.code_com
tmp = df[['id','geom']]
gpd.sjoin(df,parc[['par_id','geom']],how='left',op='intersects')
# df['num_parcelle'] =
df['surf_ore_m2'] = 0
df['date_debut_ore'] = None
df['date_fin_ore'] = None
lst_doc = []
df.loc[df.id.isin(lst_doc),'doc_foncier'] = 1
df.loc[~df.id.isin(lst_doc),'doc_foncier'] = 0
df.loc[df.doc_foncier == 1,'source_doc_foncier'] = 'CEN Isère'
df['pour_part'] = 0
df['domaine_public'] = 0
df['id_proprietaire'] = 0
df['parc_gestion_rnx'] = None # ?????????????????
df['source_geom_parc_nature'] = 2
df['source_geom_parc_annee'] = 2017
df['source_surf_parc'] = 1
# 3800010000A0001
# 38566AK00981000
# 596060000A0012
# 073490000C0103
###########
### Autre #
###########
df = pd.DataFrame()
df['id_cen'] = 14
df['id_site_cen'] = ''
0_FONCIER/TdB_FEDE/parcelles_TdB_fede2.py
+373
View File
@@ -0,0 +1,373 @@
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
from pycen import con_bdcen, con_cad
from os import getlogin
from os.path import join
from sqlalchemy.engine import URL
from sqlalchemy import create_engine
import geopandas as gpd
import pandas as pd
user = 'postgres'
pwd = 'postgres'
adr = '172.17.0.2'
base = 'postgres'
url = URL.create('postgresql+psycopg2',
username=user,
password=pwd,
host=adr,
database=base,
)
con_vm = create_engine(url)
annee = 2023
PATH = '/media/{login}/SRV/FICHIERS/OUTILS/BASES DE DONNEES/BILAN_FEDE_CEN/{an}/'
path_source = join(PATH,'Docs_SIG_joints{an}/').format(login=getlogin(),an=annee)
dico_data = 'Dico_DATA_sites_CEN_v%i_v1.xlsx' % annee
sheet_par = 'parcelles_cen_xx_%i' % annee
sheet_sit = 'sites_cen_xx_%i' % annee
# lv_cen = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='A:D',nrows=25)
# lv_typmaitr = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='F:K',nrows=23)
# lv_typmilie = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='M:N',nrows=16)
# lv_echelnum = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='P:Q',nrows=8)
# lv_typhab = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=13, usecols='P:Q',nrows=5)
# lv_sourcegeom = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='S:U',nrows=12)
# lv_lienres = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=20, usecols='M:N',nrows=4)
# lv_typprop = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=45, usecols='A:H',nrows=12)
# lv_codgeol = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=36, usecols='M:T',nrows=19)
lst_cols_parc = pd.read_excel(path_source+dico_data,sheet_name=sheet_par,header=0, usecols='F',nrows=26)\
['nom du champ'].tolist()
lst_prop = ['Acquis par le CEN ou CREN','ACQUIS&CONVENTION', 'PARTIELLEMENT MAITRISE','ACQUIS PAR LE CEN EN BND','ACQUIS PAR LE CEN ISERE']
lst_conv = ["CONVENTION D'USAGE",'CONVENTION']
lst_gere = ['BAIL EMPHYTEOTIQUE', *lst_conv, *lst_prop]
DICT_COLS = {
'id' : 'id_site_cen_parc',
'par_id' : 'num_parcelle',
'section': 'num_section',
'code_dep':'insee_dep',
'code_com':'insee_com',
}
def update_idproprietaire(df):
if 'id_proprietaire' in df.columns :
df['id_proprietaire'] = 0
df.loc[df.classif_prop.str.contains('CEN RA'),'id_proprietaire'] = 28
df.loc[df.classif_prop.str.contains('CEN ISERE'),'id_proprietaire'] = 14
df.id_proprietaire.fillna(0,inplace=True)
return df
def update_typeprop(df):
DICT = {
'Etat':'2_1_PUB',
'CEN RA - BND':'3_CEN',
'CEN RA':'3_CEN',
'CEN ISERE - BND':'3_CEN',
'Privé et entreprises':'1_PRI',
'Communes et comcom':'2_4_COM',
'Autres public et assimiliés (syndicats)':'2_0_MIX',
'Département':'2_3_PUB',
'Autres asso et fédérations':'2_6_PRI',
'FRUP':'3_CEN',
}
df['type_prop'] = df['classif_prop'].copy()
df['type_prop'].replace(DICT, inplace=True)
df.loc[
(df.classif_prop=='Privé et entreprises') &
(df.gtoper.str.contains('2')),
'type_prop'] = '2_0_MIX'
df.loc[
df.ccogrm == 9,
'type_prop'] = '2_9_PUB'
return df
def update_codemfu(df):
df.loc[df.maitrise.isin(lst_prop),'code_mfu1'] = 'P'
df.loc[df.maitrise.isin(lst_conv),'code_mfu1'] = 'C'
df.loc[df.maitrise == 'BAIL EMPHYTEOTIQUE','code_mfu1'] = 'L'
df.loc[df.maitrise.isin(lst_prop),'code_mfu2'] = 'P1'
df.loc[df.maitrise.isin(lst_conv),'code_mfu2'] = 'C7'
df.loc[df.maitrise == 'BAIL EMPHYTEOTIQUE','code_mfu2'] = 'L1'
df.loc[df.maitrise == 'ACCORD VERBAL','code_mfu1'] = 'C'
df.loc[df.maitrise == 'ACCORD VERBAL','code_mfu2'] = 'C17'
df.loc[df.maitrise == 'CONVENTION','code_mfu1'] = 'C'
df.loc[df.maitrise == 'CONVENTION','code_mfu2'] = 'C7'
df.loc[df.indivision=='USUFRUIT','code_mfu2'] = 'P2'
return df
def update_bnd(df):
df.loc[df.maitrise == 'PARTIELLEMENT MAITRISE','bnd'] = 1
df.loc[df.maitrise != 'PARTIELLEMENT MAITRISE','bnd'] = 0
df.loc[df.maitrise == 'ACQUIS&CONVENTION','bnd'] = 1
return df
def get_parc_milli():
sqlsit = '''
SELECT * FROM %s.%s
WHERE (date_fin is NULL OR date_fin >= '%i-01-01') AND type_site='MILI' ;'''%('sites','c_sites_zonages',annee)
df1 = gpd.read_postgis(sql=sqlsit,geom_col='geom',con=con_bdcen)
sql0 = '''SELECT
t1.geo_parcelle, substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) par_id,
t1.annee, t1.update_dat, t4.ccopre, t4.ccosec, t4.dcntpa, t1.geom, STRING_AGG(t3.gtoper::text,',') gtoper, STRING_AGG(t3.ccogrm::text,',') ccogrm,
STRING_AGG(CONCAT(t3.gtoper::text||COALESCE('_'||t3.ccogrm::text,'')),',') type_prop, STRING_AGG(t3.ddenom::text,',') ddenom
FROM "{sch}"."{t1}" t1
LEFT JOIN ("{sch}".parcelle_info t2
LEFT JOIN "{sch}".proprietaire t3 USING (comptecommunal))
USING (geo_parcelle)
LEFT JOIN "{sch}".parcelle t4 ON t1.geo_parcelle = t4.parcelle
'''.format(
sch='38_%i07'%(annee-1),
t1='v_geo_parcelle')
sql1 = '''
WHERE ST_INTERSECTS(t1.geom,'SRID={epsg};{poly}')
GROUP BY 1,2,3,4,5,6,7,8
'''.format(epsg=2154,poly=df1.unary_union.buffer(-1))
sql = sql0 + sql1
df2 = gpd.read_postgis(sql,con_cad)
df2['code_dep'] = df2.par_id.str[:2]
df2['code_com'] = df2.par_id.str[2:5]
df2['code_mfu1'] = 'C'
df2['code_mfu2'] = 'C12'
df2['section'] = df2.ccosec.copy()
df2['id_site_cen_parc'] = 0
df2['bnd'] = 0
df2['mesure_compens'] = 0
df2['pour_part'] = 0
return df2
def update_mc(df):
sqlsit = '''
SELECT * FROM %s.%s
WHERE (date_fin is NULL OR date_fin >= '%i-01-01') AND type_site='MC' ;'''%('sites','c_sites_zonages',annee)
df1 = gpd.read_postgis(sql=sqlsit,geom_col='geom',con=con_bdcen)
is_mc = df.intersects(df1.unary_union)
df['mesure_compens'] = 0
df.loc[
(is_mc & df.maitrise.isin(lst_conv)
)|(df.Parc_MCE=='OUI'),
'mesure_compens'
] = 1
return df
def update_pourpart(df):
df['pour_part'] = 0
df.loc[
df.maitrise=='PARTIELLEMENT MAITRISE',
'pour_part'
] = 1
return df
def is_rnx(df):
sqlsit = '''
SELECT * FROM %s.%s
WHERE (date_fin is NULL OR date_fin >= '%i-01-01') AND type_site='RNN' ;'''%('sites','c_sites_zonages',annee)
df1 = gpd.read_postgis(sql=sqlsit,geom_col='geom',con=con_bdcen)
isrnx = df.intersects(df1.unary_union.buffer(-1))
df['parc_gestion_rnx'] = 0
df.loc[isrnx,'parc_gestion_rnx'] = 1
return df
def update_datemajparcelle(df,cad):
tmp = df.merge(
parcx[['par_id','update_dat']],
how='left',
left_on='num_parcelle',
right_on='par_id')
tmp.rename(columns={'update_dat':'date_maj_parcelle'},inplace=True)
return tmp
def modif_FRUP2023(df):
df = df[
~df.commentair.str.contains(
# 'sign&2023|acq&2023',
r'sign.*2023|acq.*2023',
na=False,
case=False,
regex=True)]
is_frup = df.maitrise=='FRUP'
df.loc[is_frup,'classif_prop'] = 'CEN ISERE'
df.loc[is_frup,'maitrise'] = 'ACQUIS PAR LE CEN ISERE'
return df
def complete_tab(df):
df['id_site_cen_parc'] = df['id_site_cen_parc'].astype(int)
df['id_site_fcen_parc'] = None
df['bnd'] = df['bnd'].astype(int)
df.loc[df.type_prop.str.contains(','),'type_prop'] = '2_0'
df['surf_ore_m2'] = 0
df['date_debut_ore'] = None
df['date_fin_ore'] = None
df['doc_foncier'] = 1
df['source_doc_foncier'] = 0
df['source_geom_parc_nature'] = 2
df['source_geom_parc_annee'] = annee - 1
df['echelle_num_parc'] = 0
df['source_surf_parc'] = 1
df = update_datemajparcelle(df,parcx)
df['domaine_public'] = 0
return df
if __name__ == "__main__":
shplastsend = gpd.read_file(
join(PATH.format(login=getlogin(),an=annee-1),'ENVOIE','Parcelles_CEN_38_2022.shp')
)
lastsend = pd.read_csv(PATH.format(login=getlogin(),an=annee-1)+'ENVOIE/Parcelles_CEN_38_2022.csv')
sql = '''
SELECT * FROM foncier.c_anim_foncier_V2
WHERE maitrise not in ('Acquis par le CDI','Acquis par la commune ou Comcom','CONVENTION DE PASSAGE') AND maitrise is not NULL
; '''
dfparc = gpd.read_postgis(sql,con_bdcen)
if annee == 2023:
dfparc = modif_FRUP2023(dfparc)
dfparc = update_codemfu(dfparc)
dfparc = update_bnd(dfparc)
dfparc = update_pourpart(dfparc)
dfparc = update_mc(dfparc)
dfparc = is_rnx(dfparc)
dfparc = update_idproprietaire(dfparc)
dfparc = update_typeprop(dfparc)
dfparc['surf_parc_maitrise_m2'] = dfparc.area
# COLUMNS :
# Index(['par_id', 'geom', 'dcntpa', 'typprop_id', 'ccopre', 'section', 'dnupla',
# 'nom_com', 'numero', 'code_dep', 'code_com', 'gtoper', 'ccogrm',
# 'type_prop', 'nom_proprio', 'id', 'Parc_MCE', 'docs_off_num',
# 'Nbr_lots_BND', 'partiel_conv_ou_acq', 'code_mfu1', 'code_mfu2', 'bnd',
# 'mesure_compens', 'pour_part', 'par_id_v2', 'maitrise', 'date_der_c',
# 'indivision', 'contact', 'avis_propr', 'commentair', 'nom_proprio_old',
# 'site_id', 'type_zone', 'last_upd_parc', 'classif_prop', 'date_debut',
# 'date_fin'],
# dtype='object')
#
tmp = dfparc.rename(columns=DICT_COLS).columns.isin(lst_cols_parc)
lst_col = dfparc.rename(columns=DICT_COLS).columns[tmp]
parc = dfparc.rename(columns=DICT_COLS)[lst_col].copy()
shp_parc = dfparc[['par_id','geom']].rename(columns=DICT_COLS).copy()
sql0 = '''SELECT
t1.geo_parcelle, substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) par_id,
t1.annee, t1.update_dat, t4.ccopre, t4.ccosec, t4.dcntpa, t1.geom, STRING_AGG(t3.gtoper::text,',') gtoper, STRING_AGG(t3.ccogrm::text,',') ccogrm,
STRING_AGG(CONCAT(t3.gtoper::text||COALESCE('_'||t3.ccogrm::text,'')),',') type_prop, STRING_AGG(t3.ddenom::text,',') ddenom
FROM "{sch}"."{t1}" t1
LEFT JOIN ("{sch}".parcelle_info t2
LEFT JOIN "{sch}".proprietaire t3 USING (comptecommunal))
USING (geo_parcelle)
LEFT JOIN "{sch}".parcelle t4 ON t1.geo_parcelle = t4.parcelle
'''.format(
sch='38_202207',
t1='v_geo_parcelle')
sql1 = '''WHERE substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) IN ('%s')
GROUP BY 1,2,3,4,5,6,7,8'''%"','".join(dfparc.par_id.tolist())
sql = sql0 + sql1
parcx = gpd.read_postgis(sql,con=con_cad)
noparcx = dfparc[~dfparc.par_id.isin(parcx.par_id)]
newparc = parc[~parc.num_parcelle.isin(lastsend.num_parcelle)].copy()
shp_newparc = shp_parc[~shp_parc.num_parcelle.isin(lastsend.num_parcelle)].copy()
newparcx = parcx[parcx.par_id.isin(shp_newparc.num_parcelle)].copy()
shp_newparc = shp_newparc.merge(newparcx[['par_id','update_dat']],how='left',left_on='num_parcelle',right_on='par_id')\
.drop(columns=['par_id'])\
.rename(columns={
'num_parcelle':'num_parc',
'update_dat':'date_maj_p'
})\
.rename_geometry('geometry')
newparc = complete_tab(newparc)
parc = complete_tab(parc)
# mili = get_parc_milli()
PARC = pd.concat([lastsend,newparc])
PARC.id_site_cen_parc = PARC.id_site_cen_parc.astype(int)
# Mise au format des dates
date_cols = PARC.columns[PARC.columns.str.contains('date')]
for c in date_cols:
PARC[c] = pd.to_datetime(PARC[c],yearfirst=True).dt.strftime('%Y/%m/%d')
# site2023[c] = site2023[c].dt.strftime('%Y/%m/%d')
SHP_PARC = gpd.GeoDataFrame(
pd.concat([shplastsend,shp_newparc]),
geometry='geometry',
crs=2154
)
SHP_PARC['date_maj_p'] = pd.to_datetime(SHP_PARC['date_maj_p'],yearfirst=True).dt.strftime('%Y/%m/%d')
SHP_PARC.geometry = SHP_PARC.buffer(0)
PARC.to_csv(
join(
PATH.format(login=getlogin(),an=annee),
'Recueil_data',
'parcelles_cen_38_2023.csv'),
index=False)
SHP_PARC.to_file(
join(
PATH.format(login=getlogin(),an=annee),
'Recueil_data',
'parcelles_cen_38_2023.shp'),
index=False)
0_FONCIER/TdB_FEDE/site_TdB_2024_complete.py
+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
from pycen import con_fon
import pandas as pd
dico_data = '/media/cgeier/SRV/FICHIERS/OUTILS/BASES DE DONNEES/BILAN_FEDE_CEN/2024/TDB2024_enquete_SIG/Dico_DATA_sites_CEN_v2024.xlsx'
bilan_2023 = '/media/cgeier/SRV/FICHIERS/OUTILS/BASES DE DONNEES/BILAN_FEDE_CEN/2024/TDB2024_enquete_SIG/DATA N-1/Sites_CEN_38_2023.csv'
dic = pd.read_excel(dico_data,sheet_name='sites_cen_xx_2024',header=0, usecols='F',nrows=50)
dic_head_name = dic.columns[0]
bil2023 = pd.read_csv(bilan_2023,sep=',',header=0,encoding='utf-8')
# Mise en forme des dates
date_cols = bil2023.columns[bil2023.columns.str.contains('date')]
bil2023[date_cols] = bil2023[date_cols].apply(pd.to_datetime)
# Mise en forme de la colonne remq_sensibilite
vm_site = pd.read_sql_table('vm_sites_cen_2024_csv',con_fon,'_tdbfcen')
dic_missing = (dic.loc[~dic[dic_head_name].isin(vm_site.columns),dic_head_name]
.tolist())
(bil2023[['id_site_cen',*dic_missing]]
.merge(vm_site[['id_site_cen']],how='inner',on='id_site_cen')
.to_sql(
'_sites_cen_2023_csv_complement',
con_fon,
schema='_tdbfcen',
if_exists='replace',
index=False))
0_FONCIER/TdB_FEDE/sites_TdB_fede.py
+389
View File
@@ -0,0 +1,389 @@
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
from os import getlogin,path,chdir
import sys
import pandas as pd
import geopandas as gpd
from sqlalchemy.engine import URL
from sqlalchemy import create_engine
from pycen import con_bdcen as con, con_fon, wfs
# NOT WORK
# SCRIPT_DIR = path.dirname(path.abspath(__file__))
# sys.path.append(path.dirname(SCRIPT_DIR))
# from . import get_siteCen_fede
annee = 2023
term_parcelle_conv = ['CONVENTION',"CONVENTION D'USAGE",'ACCORD VERBAL']
path_agri = '/media/cgeier/SRV/FICHIERS/OUTILS/CARTOGRAPHIE/ESPACE DE TRAVAIL/TRAVAUX/Couches de référence/'
agri_partenaire_surf = gpd.read_file(path_agri+'AGRIS_partenaires_surface exploitées.shp')
agri_partenaire = gpd.read_file(path_agri+'AGRIS_partenaires.shp')
id_rnx = ['RNGL','RNIP']
MAJ_COLNAME = {
'doc_gestion_annee_ini':'doc_gestion_date_ini',
'doc_gestion_annne_maj':'doc_gestion_date_maj',
'source_geom_site_annee':'source_geom_site_date',
'0':'surf_carto_habitat_m2',
'F':'geol_site_inpn',
}
def drop_specialchar(obj):
return obj.replace({
r'[’]':"'",
r'[àáâãäå]':'a',
r'[èéêë]':'e',
r'[ìíîï]':'i',
r'[òóôõö]':'o',
r'[ùúûü]':'u',
# r'[ ]':"",
r'[–]':"-"
},regex=True)
def get_site():
sqlsit = '''SELECT * FROM %s.%s'''%('sites','c_sites_zonages')
return gpd.read_postgis(sql=sqlsit,geom_col='geom',con=con)
def get_siteCen_fede(an=None):
sqlsit = '''SELECT * FROM %s.%s'''%('sites','c_sites_zonages')
if an :
sqlsit += ''' WHERE date_fin is NULL OR date_fin >= '%i-01-01' '''%an
sqlsit += ' order by code_site'
dfsite = gpd.read_postgis(sql=sqlsit,geom_col='geom',con=con)
# Isolation de tous les sites CEN (gestion/assist/mili)
dup = dfsite.code_site.duplicated(keep=False)
typzon = dfsite.type_zonage.isin(['ZI','DO','DH'])
return dfsite[
~dup | (dup & typzon)
]
def get_parcelleCen_fede(an=None):
sqlsit = '''SELECT * FROM %s.%s'''%('foncier','c_anim_foncier_v2')
if an :
sqlsit += '''
WHERE maitrise IN (
'ACCORD VERBAL','ACQUI PAR LE CEN EN BND','ACQUI PAR LE CEN ISERE',
'ACQUIS PAR LE CEN ISERE','ACQUIS&CONVENTION','Acquis par le CEN ou CREN',
'BAIL EMPHYTEOTIQUE','CONVENTION','CONVENTION D''USAGE','FRUP')
AND (commentair not ilike '%%{an}%%' OR commentair IS NULL)'''.format(an=an)
return gpd.read_postgis(sql=sqlsit,geom_col='geom',con=con)
def update_date_site2022(df):
df['date_crea_site'].replace({
'01/012005': '01/01/2005',
'01/01/98': '01/01/1998',
'01/01/03': '01/01/2003',
'09.05.2001': '09/05/2001',
'03/1986': '01/03/1986',
'01/01/94': '01/01/1994',
}, inplace=True)
df['date_crea_site'] = pd.to_datetime(
df['date_crea_site'],
# format='%Y/%m/%d',
yearfirst=True
)
return df
def update_date(serie):
DICT = {
'024/04/2019':'24/04/2019'
}
return pd.to_datetime(
serie.replace(DICT),
# format='%Y/%m/%d',
yearfirst=True
)
if __name__ == "__main__":
path_source = '/media/{login}/SRV/FICHIERS/OUTILS/BASES DE DONNEES/BILAN_FEDE_CEN/{an}/'.format(login=getlogin(),an=annee)
path_an_precedent = '/media/{login}/SRV/FICHIERS/OUTILS/BASES DE DONNEES/BILAN_FEDE_CEN/{an}/ENVOIE/'.format(login=getlogin(),an=annee-1)
tdb_sites_csv = 'Sites_CEN_38_%i.csv'
receuil_path = 'Recueil_data/'
receuil_datasite = 'TBD - receuil data SITES.xlsx'
shp_cnpe = 'primtresitecnpe/perimetre_site_CNPE_2019.shp'
# path_source = '/media/{login}/SRV/FICHIERS/OUTILS/BASES DE DONNEES/BILAN_FEDE_CEN/{an}/Docs_SIG_joints{an}/'.format(login=getlogin(),an=annee)
dico_data = 'Docs_SIG_joints{an}/Dico_DATA_sites_CEN_v{an}_v1.xlsx'.format(an=annee)
lv_typmilie = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs',header=1, usecols='M:N',nrows=16)
lv_typmilie.columns = ['code_type_milieu','libelle_type_milieu']
dfsite = get_siteCen_fede(annee-1)\
.sort_values('code_site')\
.reset_index()
dfsite.loc[(dfsite.code_site=='N2IP') & (dfsite.type_zonage=='DO'),'code_site'] = 'N2IP_DO'
dfsite.loc[(dfsite.code_site=='N2IP') & (dfsite.type_zonage=='DH'),'code_site'] = 'N2IP_DH'
dfparc = get_parcelleCen_fede(annee)
sitefcen = dfsite[dfsite.intersects(dfparc.unary_union)]
parconv = dfparc.maitrise.isin(term_parcelle_conv)
site_rnx = dfsite[dfsite.code_site.isin(id_rnx)].copy()
n2000_zps = wfs.get_wfs(
url = "https://ws.carmencarto.fr/WMS/119/fxx_inpn?",
layer = "Zones_de_protection_speciale",
bbox = dfsite.unary_union )
# n2000_zps = wfs.get_wfs(url="https://ws.carmencarto.fr/WMS/119/fxx_inpn?",layer="Zones_de_protection_speciale")
n2000_zsc = wfs.get_wfs(
url="https://ws.carmencarto.fr/WMS/119/fxx_inpn?",
layer="Sites_d_importance_communautaire_JOUE__ZSC_SIC_",
bbox=dfsite.unary_union )
site2022 = pd.read_csv(path_an_precedent+tdb_sites_csv % (annee-1))\
.rename(columns=MAJ_COLNAME)\
.replace({'id_site_cen':{'COMB':'COCO'}})
site2022 = update_date_site2022(site2022)
site2023 = site2022[site2022.id_site_cen.isin(dfsite.code_site)]\
.reset_index(drop=True)
recueil_site = pd.read_excel(path_source+receuil_path+receuil_datasite,header=0,skiprows=[1])\
.replace({'Null':None})
recueil_site['doc_gestion_date_ini'] = update_date(recueil_site['doc_gestion_date_ini'])
recueil_site['doc_gestion_date_maj'] = update_date(recueil_site['doc_gestion_date_maj'])
recueil_site['doc_gestion_date_fin'] = update_date(recueil_site['doc_gestion_date_fin'])
test_add = ~recueil_site.id_site.isin(site2023.id_site_cen)
add_site = recueil_site[test_add].rename(columns={
'id_site':'id_site_cen'
})
site2023 = pd.concat([site2023,add_site])
# site_exploit_agri = dfsite.intersects(agri_partenaire.unary_union)
site_exploit_agri_surf = dfsite.intersects(agri_partenaire_surf.unary_union)
# Test d'identification des nouveaux sites
sitenew = ~dfsite.code_site.isin(site2022.id_site_cen)
if sitenew.any():
print('WARNING : de nouveaux sites sont apparus cette année :\n\t%s'%str(tuple(dfsite[sitenew].code_site)))
intersect_acqu = dfsite[sitenew].intersects(dfparc[~parconv].unary_union)
intersect_conv = dfsite[sitenew].intersects(dfparc[parconv].unary_union)
if intersect_acqu.any():
print(' Le(s) nouveau(x) site(s) intersect(ent) des parcelles acquises : \n\t%s'%str(tuple(dfsite[sitenew&intersect_acqu].code_site)))
else:
print(' Le(s) nouveau(x) site(s) ne contien(nen)t pas de parcelles acquises ...')
if intersect_conv.any():
print(' Le(s) nouveau(x) site(s) intersect(ent) des parcelles conventionnées : \n\t%s'%str(tuple(dfsite[sitenew&intersect_conv].code_site)))
else:
print(' Le(s) nouveau(x) site(s) ne contien(nen)t pas de parcelles conventionnées ...')
dfsite = dfsite[~sitenew].copy()\
.reset_index()
else :
print("Pas de nouveaux sites pour l'année %i."%annee)
site2023.sort_values('id_site_cen',inplace=True)
recueil_site.sort_values('id_site',inplace=True)
dfsite.sort_values('code_site',inplace=True)
# nom_site
site2023['nom_site'] = [
dfsite[dfsite.code_site==x].nom_site.values[0]
if x in [*dfsite.code_site] else None
for x in site2023.id_site_cen
]
site2023['nom_site'] = drop_specialchar(site2023['nom_site'].str.lower())
site2023['nom_site'] = site2023['nom_site'].str.upper()
# site_rnx_surface_m2
site2023.site_rnx_surface_m2 = dfsite\
.intersection(site_rnx.unary_union)\
.area.astype(int)
# ens (A FAIRE)
site2023['ens'] = site2023['ens'].fillna(1).astype(int)
# site_cdl
site2023.site_cdl = 0
# n2000_directive
site2023 = site2023.merge(dfsite[['code_site','geom']],how='left',left_on='id_site_cen',right_on='code_site')\
.drop(columns=['code_site'])
cnpe = gpd.read_file(path_source+receuil_path+shp_cnpe)
# site2023.loc[site2023.id_site_cen=='CNPE','geom'] = cnpe.iloc[0].geometry
site2023 = gpd.GeoDataFrame(site2023,geometry='geom',crs=2154)
site2023['is_zps'] = site2023.intersects(n2000_zps.unary_union)
site2023['is_zsc'] = site2023.intersects(n2000_zsc.unary_union)
site2023['n2000'] = site2023[['is_zps','is_zsc']].sum(axis=1)
site2023.loc[site2023.n2000==2,'n2000_directive'] = 'ZPS_ZSC'
site2023.loc[site2023.n2000==0,'n2000_directive'] = '0'
site2023.loc[site2023.is_zps & (site2023.n2000!=2),'n2000_directive'] = 'ZPS'
site2023.loc[site2023.is_zsc & (site2023.n2000!=2),'n2000_directive'] = 'ZSC'
site2023.drop(columns=['is_zps','is_zsc','n2000'],inplace=True)
# n2000_surface_m2
site2023['n2000_surface_m2'] = site2023\
.intersection(pd.concat([n2000_zps,n2000_zsc]).unary_union)\
.area.astype(int)
# terrain_militaire
site2023['terrain_militaire'] = (dfsite.type_site == 'MILI').astype(int)
# site_marin
site2023['site_marin'] = 0
# nb_contrat_agri
site2023['nb_contrat_agri'] = site2023['nb_contrat_agri'].fillna(0)
# nb_agri
site2023['nb_agri'] = site2023['nb_agri']\
.replace({'?':0})\
.fillna(0)\
.astype(int)
# surf_contra_m2
site2023['surf_contra_m2'] = site2023['surf_contra_m2'].fillna(0)
# code_milieu_princ
dict_milieu = dict(zip(lv_typmilie.libelle_type_milieu,lv_typmilie.code_type_milieu))
dfsite['milieux'].replace({'Gîtes à chiroptères et milieux souterrains':'Gîtes à chiroptères'}, inplace=True)
dfsite['code_milieu'] = dfsite.milieux.str.lower().replace(dict_milieu)
site2023['code_milieu_princ'] = [
dfsite[dfsite.code_site==x].code_milieu.values[0] if x in [*dfsite.code_site] else None for x in site2023.id_site_cen
]
# nature_site_inpn
site2023['nature_site_inpn'] = 'N'
# geol_site_inpn
site2023['geol_site_inpn'] = 'N'
# code_geol
site2023['code_geol'] = 21
# carto_habitats
site2023['carto_habitats'] = site2023['carto_habitats'].fillna(0)
# typo_carto_habitat
site2023['typo_carto_habitat'] = site2023['typo_carto_habitat'].fillna(0)
# surf_carto_habitat_m2
site2023['surf_carto_habitat_m2'] = site2023['surf_carto_habitat_m2'].fillna(0)
# date_crea_site
datemiss = site2023['date_crea_site'].isnull()
for s in site2023[datemiss].id_site_cen:
if s not in dfsite.code_site : pass
site2023.loc[site2023.id_site_cen==s,'date_crea_site'] = dfsite[dfsite.code_site==s].date_ajout
site2023['date_crea_site'] = pd.to_datetime(site2023['date_crea_site'],yearfirst=True, format='%Y/%m/%d')
# date_maj_site
site2023['date_maj_site'] = dfsite.date_maj.copy()
# nature_perimetre
site2023['nature_perimetre'] = site2023['nature_perimetre'].fillna(0)
# source_geom_site_nature
site2023['source_geom_site_nature'] = site2023['source_geom_site_nature'].fillna(0)
# source_geom_site_date
site2023['source_geom_site_date']
# echelle_num_site
site2023['echelle_num_site'] = site2023['echelle_num_site'].fillna(0)
# precision_num_site
site2023['precision_num_site'] = site2023['precision_num_site'].fillna('NE')
# gestionnaire_site
gestmiss = site2023['gestionnaire_site'].isna()
site2023.loc[gestmiss,'gestionnaire_site'] = 'CEN Isère'
# operateur
opmiss = site2023['operateur'].isna()
site2023.loc[opmiss,'operateur'] = 'FCEN'
# surf_libre_evolution_m2
site2023['surf_libre_evolution_m2'] = site2023['surf_libre_evolution_m2'].fillna(0)
# doc_gestion_presence
site2023['doc_gestion_presence'] = site2023['doc_gestion_presence'].fillna(0)
# doc_gestion_nom
site2023['doc_gestion_nom']
# doc_gestion_evaluation
site2023['doc_gestion_evaluation']
# doc_gestion_date_ini
# site2023['doc_gestion_date_ini'] = pd.to_datetime(site2023['doc_gestion_date_ini'],yearfirst=True)
site2023['doc_gestion_date_ini'] = update_date(site2023['doc_gestion_date_ini'])
# doc_gestion_date_maj
# site2023['doc_gestion_date_maj'] = pd.to_datetime(site2023['doc_gestion_date_maj'],yearfirst=True)
site2023['doc_gestion_date_maj'] = update_date(site2023['doc_gestion_date_maj'])
# doc_gestion_date_fin
# site2023['doc_gestion_date_fin'] = pd.to_datetime(site2023['doc_gestion_date_fin'],yearfirst=True)
site2023['doc_gestion_date_fin'] = update_date(site2023['doc_gestion_date_fin'])
# surf_doc_gestion_m2
site2023['surf_doc_gestion_m2'] = site2023['surf_doc_gestion_m2'].fillna(0)
# url_fiche_inpn
site2023['url_fiche_inpn']
# url_fiche_cen
site2023['url_fiche_cen']
# doc_justif_admin
site2023['doc_justif_admin']
# ouverture_public
site2023['ouverture_public']
# description_site
site2023['description_site']
# url_site_photo
site2023['url_site_photo']
# sensibilite
site2023['sensibilite']
# remq_sensibilite
site2023['remq_sensibilite']
site2023.sort_values('id_site_cen',inplace=True)
recueil_site.sort_values('id_site',inplace=True)
for c in recueil_site.columns:
if c not in site2023.columns: pass
notna = recueil_site[c].notna()
lstsite = recueil_site[notna].id_site.tolist()
s_2023 = site2023.id_site_cen.isin(lstsite)
r_2023 = recueil_site.id_site.isin(lstsite)
site2023.loc[s_2023,c] = recueil_site[r_2023][c]
# test_add = ~recueil_site.id_site.isin(site2023.id_site_cen)
# add_site = recueil_site[test_add].rename(columns={
# 'id_site':'id_site_cen'
# })
# SITE2023 = pd.concat([site2023,recueil_site[test_add]])
for c in site2023.columns:
if site2023[c].dtype==float:
try:
site2023[c] = site2023[c].astype(int)
except:
pass
# Mise au format des dates
date_cols = site2023.columns[site2023.columns.str.contains('date')]
for c in date_cols:
site2023[c] = pd.to_datetime(site2023[c],yearfirst=True).dt.strftime('%Y/%m/%d')
# site2023[c] = site2023[c].dt.strftime('%Y/%m/%d')
shp2023 = site2023[['id_site_cen','id_site_fcen','date_crea_site','date_maj_site','geom']].copy()
shp2023['date_maj_s'] = [
shp2023.date_maj_site[i] if pd.notna(shp2023.date_maj_site[i]) else shp2023.date_crea_site[i] for i in shp2023.index
]
shp2023.drop(columns=['date_crea_site','date_maj_site'],inplace=True)
shp2023.rename(columns={'id_site_fcen':'id_fcen'},inplace=True)
# Ecriture des fichiers finaux
site2023.drop(columns=['geom','id_site','gestionnaire']).to_csv(path_source+receuil_path+tdb_sites_csv%annee,index=False)
shp2023.to_file(
(path_source+receuil_path+tdb_sites_csv[:-3]+'shp')%annee,
index=False)
# from pycen import update_to_sql,
# site = get_site()
# darp = site.code_site.isin(['DARN','PEYR'])
# zi = site.type_zonage=='ZI'
# zo = site.type_zonage=='ZO'
# DARP = gpd.GeoDataFrame({
# 'code_site':['DARP','DARP'],
# 'type_zonage': ['ZI','ZO'],
# },
# geometry=[
# site[darp&zi].unary_union,site[darp&zo].unary_union
# ],crs=2154)\
# .rename_geometry('geom')
# DARP['surface_ha'] = round(DARP.area / 10000,2)
# update_to_sql(cnpe,con,'c_sites_zonages','sites','code_site')
0_FONCIER/add_c_anim_foncier_v2.py
+91
View File
@@ -0,0 +1,91 @@
from pycen import con_bdcen,con_fon,update_to_sql
import geopandas as gpd
from datetime import datetime as dt
# GET parcelles gérées à jour
sql = "SELECT * FROM sites.c_sites_zonages WHERE code_site = 'LEZE'"
sql = "SELECT * FROM foncier.c_anim_foncier_v2"
parc_site = gpd.read_postgis(sql,con_bdcen)
parc_site[(~parc_site.classif_prop.str.contains('cen',case=False))&(parc_site.maitrise.str.contains('acqui',case=False))].to_csv('/media/cgeier/SRV/FICHIERS/TRANSFERTS-EQUIPE/AB/verif_anim_foncier_v2.csv',index=False)
# parc_site.loc[parc_site.maitrise=='ACQUI PAR LE CEN ISERE','maitrise'] = 'ACQUIS PAR LE CEN ISERE'
# update_to_sql(
# parc_site[['par_id','maitrise']],con_bdcen,'c_anim_foncier_v2','foncier','par_id'
# )
# GET sites
sql = "SELECT * FROM sites.sites"
site_fon = gpd.read_postgis(sql,con_fon)
site_zi = site_fon[site_fon.site_id.str.endswith('ZI')]
site_zo = site_fon[site_fon.site_id.str.endswith('ZO')]
# GET cadastre from bd_cen
sql = '''
WITH prop AS (
SELECT
dnuper,gtoper,ccogrm,ddenom
FROM {sch}."proprios"
ORDER BY 2,3,4
)
SELECT --DISTINCT ON (t1.par_id,t1.geom, t1.dcntpa,t1.typprop_id,t1.ccopre,t1.ccosec,t1.dnupla)
t1.par_id,
t1.geom,
t1.dcntpa,
t1.typprop_id,
t1.ccopre,
t1.ccosec "section",
t1.dnupla::text,
c.nom nom_com,
substring(t1.par_id from 12 for 4) "numero",
substring(t1.codcom from 1 for 2) code_dep,
substring(t1.codcom from 3 for 3) code_com,
STRING_AGG(distinct t6.gtoper::text,',') gtoper,
STRING_AGG(distinct t6.ccogrm::text,',') ccogrm,
STRING_AGG(distinct CONCAT(TRIM(t6.gtoper::text)||COALESCE('_'||TRIM(t6.ccogrm::text),'')),',') type_prop,
--t6.dnuper,
STRING_AGG(distinct t6.ddenom::text,', ') nom_proprio
FROM {sch}."{t1}" t1
JOIN ({sch}."lots" t2
JOIN ({sch}."cadastre" t3
JOIN ({sch}."cptprop" t4
JOIN ({sch}."r_prop_cptprop" t5
JOIN prop t6 USING (dnuper))
USING (dnupro))
USING (dnupro))
USING (lot_id))
USING (par_id)
JOIN administratif.communes c on t1.codcom = c.code_insee '''.format(
sch='cadastre',
t1='parcelles'
)
sql_grp = '''GROUP BY 1,2,3,4,5,6,8'''
# Parcelles gestion
csv_parc = gpd.pd.read_csv('/home/colas/Documents/9_PROJETS/0_FONCIER/CEN38/LEZE_parcelles_a_rajouter_CSV.csv')
csv_parc['par_id'] = csv_parc.geo_parcel.str[:2]+csv_parc.geo_parcel.str[3:]
lst_parc = csv_parc.par_id.tolist()
sql_whr = "WHERE t1.par_id IN ('{par_ids}') ".format(par_ids="','".join(lst_parc))
sql_gst = sql + sql_whr + sql_grp
parc = gpd.read_postgis(sql_gst,con_fon)
parc['last_upd_parc'] = dt.now().date().isoformat()
parc.loc[parc.type_prop.str.startswith(','),'type_prop'] = parc[parc.type_prop.str.startswith(',')].type_prop.str[1:]
parc['classif_prop'] = 'Privé et entreprises'
parc.loc[parc.type_prop=='1','classif_prop'] = 'Privé et entreprises'
parc2 = parc.sjoin(site_fon[['site_id','geom']],how='left')\
.replace({'site_id':{'_ZI|_ZO':''}},regex=True)\
.drop(columns=['index_right'])\
.drop_duplicates()\
.reset_index(drop=True)
# Envoie bd-cen-38.foncier.c_anim_foncier_v2
parc2.to_postgis(
name = 'c_anim_foncier_v2',
con = con_bdcen,
schema = 'foncier',
if_exists = 'append',
index=False
)
0_FONCIER/config_insert_cadastre.py
+118
View File
@@ -0,0 +1,118 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Nom : : config_insert_table.py
#Description : Insertion des données cadastrales à la base <foncier> après de sa création.
#Copyright : 2021, CEN38
#Auteur : Colas Geier
#Version : 1.0
DICT_TAB = [{
'table_in' : 'proprietaire', # Table source qui provient de la sortie du plugin cadastre de qgis
'index_tab': 'proprietaire', # Pkey de la table source
'columns_in': ['ccodep', 'dnuper', 'ccoqua', 'ddenom', 'jdatnss', 'dldnss', 'dsglpm', 'dlign3', 'dlign4', 'dlign5', 'dlign6', 'dnatpr', 'gtoper', 'ccogrm',
'ccocom', 'dnupro', 'dnomlp', 'dprnlp', 'epxnee', 'dnomcp', 'dprncp', 'ccodro', 'ccodem'],
'table_out': [{
'name': 'cptprop{}'.format(dpt_nom_tab),
'geom': None,
'drop_escape': False, # Supprime les champs vides à l'intérieure des chaines de carractères
'columns_in': ['ccodep', 'ccocom', 'dnupro'], # Liste des columns à récupérer en entrée.
'columns_add': {'dnupro': ['ccodep', 'ccocom', 'dnupro']}, # Définition des champs composés devant être ajoutés
'unique': {'cols': ['dnupro'], 'keep': 'first'}, # Champs devant être uniques à l'intérieur de la table en sortie
'dict': None, # Dictionnaire pour renommer les champs {'ancien_nom1': 'nouveau_nom1', 'ancien_nom2': 'nouveau_nom2', ...}
'join': False
},{
'name': 'proprios{}'.format(dpt_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in': ['ccodep', 'dnuper', 'ccoqua', 'ddenom', 'jdatnss', 'dldnss', 'dsglpm', 'dlign3', 'dlign4', 'dlign5', 'dlign6', 'dnatpr', 'gtoper', 'ccogrm'],
'columns_add': {'dnuper': ['ccodep', 'dnuper']},
'unique': {'cols': ['dnuper'], 'keep': 'first'},
'dict': None,
'join': False
},{
'name': 'r_prop_cptprop{}'.format(dpt_nom_tab),
'geom': None,
'drop_escape': False,
'columns_in': ['ccodep', 'dnuper', 'ccocom', 'dnupro', 'dnomlp', 'dprnlp', 'epxnee', 'dnomcp', 'dprncp', 'ccodro', 'ccodem'],
'columns_add': {
'dnuper': ['ccodep', 'dnuper'],
'dnupro': ['ccodep', 'ccocom', 'dnupro']},
'unique': {'cols': ['dnupro', 'dnuper'], 'keep': 'first'},
'dict': None,
'join': False
},]
},{
'table_in' : 'parcelle',
'index_tab': 'parcelle',
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'ccovoi', 'dparpi', 'dcntpa', 'ccocomm', 'ccoprem', 'ccosecm', 'dnuplam', 'dvoilib', 'type_filiation'],
'table_out': [{
'name': 'vl{}'.format(dpt_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccovoi', 'dvoilib'],
'columns_add': {
'vl_id': ['ccodep', 'ccocom', 'ccovoi'],
'geom': None},
'unique': {'cols': ['vl_id'], 'keep': 'first'},
'dict': None,
'join': False
},{
'name': 'parcelles{}'.format(dpt_nom_tab),
'geom': {
'table_geom_in': 'geo_parcelle',
'index_geom': 'geo_parcelle'
},
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'ccovoi', 'dparpi', 'dcntpa', 'ccocomm', 'ccoprem', 'ccosecm', 'dnuplam', 'type_filiation'],
'columns_add': {
'par_id': ['ccodep', 'ccocom', 'ccopre','ccosec', 'dnupla'],
'codcom': ['ccodep', 'ccocom'],
'vl_id': ['ccodep', 'ccocom', 'ccovoi'],
'typprop_id': None },
'unique': False,
'dict': {'type_filiation': 'type'},
'join': False
},{
'name': 'lots{}'.format(dpt_nom_tab), # !!!!!! Ne trouve pas de parcelles sans lots (ex: 38357000AE0526)
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dcntpa'],
'columns_add': {
'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla'],
'par_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla'],
'dnulot': None, },
'unique': False,
'dict': {'dcntpa': 'dcntlo'},
'join': [{'bdd': 'out', 'table': 'parcelles{}'.format(dpt_nom_tab), 'on': ['par_id'], 'type': 'distinct'}]
},]
},{
'table_in' : 'lots',
'index_tab': 'lots',
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnupdl', 'dcntlo'],
'table_out': [{
'name': 'lots{}'.format(dpt_nom_tab), # !!!!!! parcelles avec lots: existe par_id NOT IN parcelles_73
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnupdl', 'dcntlo'],
'columns_add': {
'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'],
'par_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla'],},
'unique': {'cols': ['lot_id'], 'keep': 'first'},
'dict': None,
'join': False
},{
'name': 'lots_natcult{}'.format(dpt_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'],
'columns_add': {
'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'],},
'unique': {'cols': ['lot_id'], 'keep': 'first'},
'dict': None,
'join': [{ # ERROR ! 2 dclssf pour 1 lot_id
'bdd': 'in', 'table': 'suf', 'on': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'], 'type': 'merge',
'select_cols' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot','dsgrpf','cnatsp','dclssf','ccosub','dcntsf'],
}]
},]
}]
0_FONCIER/create_c_anim_foncier_v2.py
+187
View File
@@ -0,0 +1,187 @@
from pycen import con_bdcen,con_fon
import geopandas as gpd
from datetime import datetime as dt
# GET parcelles gérées à jour
sql = "SELECT * FROM parcelles_foncier_tofede"
parc_site = gpd.read_postgis(sql,con_bdcen)
parc_smrg = parc_site[[
'par_id','id','Parc_MCE','docs_off_num',
'Nbr_lots_BND','partiel_conv_ou_acq',
'code_mfu1','code_mfu2','bnd','mesure_compens',
'pour_part'
]]
# GET couche animation fonciere
sql = """SELECT
c.*,
substring(c.num_unique_v2 from 1 for 5) || substring(c.num_unique_v2 from 13 for 3) || substring(c.num_unique_v2 from 6 for 6) par_id_v2
FROM foncier.c_animation_fonciere_sites c"""
anim_fon = gpd.read_postgis(sql,con_bdcen)
anim_mrg = anim_fon[['par_id_v2',
'id','maitrise','date_der_c','indivision',
'contact','avis_propr','commentair','nom_propri'
]].rename(
columns={
'nom_propri':'nom_proprio_old'
}
)
# GET sites
sql = "SELECT * FROM sites.sites"
site_fon = gpd.read_postgis(sql,con_fon)
site_zi = site_fon[site_fon.site_id.str.endswith('ZI')]
site_zo = site_fon[site_fon.site_id.str.endswith('ZO')]
lst_pmili = parc_site[parc_site.id.isna()].par_id.tolist()
# 3880,3884,3885,3883,3886,3859,3860,3856,3858,3854
# par_id / lot_id
# '38222000ZB0013','38222000ZB0015','38222000ZB0016','38222000ZB0017','38222000ZB0023','38222000ZB0041','38222000ZB0042','38222000ZB0043','38222000ZB0058','38222000ZB0059'
# vl_id
# '3822200101','3822200126','3822200126','3822200126','3822200108','3822200106','3822200106','3822200106','3822200089','3822200089'
# GET cadastre from bd_cen
sql = '''
WITH prop AS (
SELECT
dnuper,gtoper,ccogrm,ddenom
FROM {sch}."proprios"
ORDER BY 2,3,4
)
SELECT --DISTINCT ON (t1.par_id,t1.geom, t1.dcntpa,t1.typprop_id,t1.ccopre,t1.ccosec,t1.dnupla)
t1.par_id,
t1.geom,
t1.dcntpa,
t1.typprop_id,
t1.ccopre,
t1.ccosec "section",
t1.dnupla::text,
c.nom nom_com,
substring(t1.par_id from 12 for 4) "numero",
substring(t1.codcom from 1 for 2) code_dep,
substring(t1.codcom from 3 for 3) code_com,
STRING_AGG(distinct t6.gtoper::text,',') gtoper,
STRING_AGG(distinct t6.ccogrm::text,',') ccogrm,
STRING_AGG(distinct CONCAT(TRIM(t6.gtoper::text)||COALESCE('_'||TRIM(t6.ccogrm::text),'')),',') type_prop,
--t6.dnuper,
STRING_AGG(distinct t6.ddenom::text,', ') nom_proprio
FROM {sch}."{t1}" t1
JOIN ({sch}."lots" t2
JOIN ({sch}."cadastre" t3
JOIN ({sch}."cptprop" t4
JOIN ({sch}."r_prop_cptprop" t5
JOIN prop t6 USING (dnuper))
USING (dnupro))
USING (dnupro))
USING (lot_id))
USING (par_id)
JOIN administratif.communes c on t1.codcom = c.code_insee '''.format(
sch='cadastre',
t1='parcelles'
)
sql_grp = '''GROUP BY 1,2,3,4,5,6,8'''
# Parcelles gestion
lst_parc = parc_site.par_id.tolist()
sql_whr = "WHERE t1.par_id IN ('{par_ids}') ".format(par_ids="','".join(lst_parc))
sql_gst = sql + sql_whr + sql_grp
parc_gst = gpd.read_postgis(sql_gst,con_fon)
parc_gst_tmp = parc_gst.merge(parc_smrg,on='par_id')
parc_gst_2 = parc_gst_tmp.merge(anim_mrg,on='id')
# parcelles sites, NON PRESENTES dans parcelles gérées
geo = site_fon.unary_union
sql_whr = """
WHERE ST_Intersects (t1.geom, 'SRID={epsg};{poly}')
AND t1.par_id NOT IN ('{par_ids}')""".format(
epsg = 2154,
poly = geo,
par_ids="','".join(lst_parc)
)
sql_veil = sql + sql_whr + sql_grp
parc_veil = gpd.read_postgis(sql_veil,con_fon)
parc_veil2 = parc_veil.merge(anim_mrg,left_on='par_id',right_on='par_id_v2',how='left')
# Parcelles Hors site
lst_id = [
*parc_veil2.id.dropna().astype(int).tolist(),
*parc_gst_2.id.dropna().astype(int).tolist()
]
lst_parid_HS = anim_fon[~anim_fon.id.isin(lst_id)].par_id_v2.dropna().tolist()
sql_whr = "WHERE t1.par_id IN ('{par_ids}') ".format(par_ids="','".join(lst_parid_HS))
sql_hs = sql + sql_whr + sql_grp
parc_hs = gpd.read_postgis(sql_hs,con_fon)
# parc_hs2 = parc_hs.merge(anim_mrg,left_on='par_id',right_on='par_id_v2')
parc_hs2 = parc_hs[parc_hs.is_valid].merge(anim_mrg,left_on='par_id',right_on='par_id_v2')
INVALID_parc_hs = parc_hs[~parc_hs.is_valid].merge(anim_mrg,left_on='par_id',right_on='par_id_v2')
# parc_gst_2[parc_gst_2.type_prop.str.startswith(',')].iloc[:,:15]
# parc_veil2[parc_veil2.type_prop.str.startswith(',')].iloc[:,:15]
# parc_hs2[parc_hs2.type_prop.str.startswith(',')]
# Merge parcelles
parc = gpd.pd.concat([parc_gst_2,parc_veil2,parc_hs2])\
.reset_index(drop=True)
parc.loc[parc.type_prop.str.startswith(','),'type_prop'] = parc[parc.type_prop.str.startswith(',')].type_prop.str[1:]
# Jointure id_site
parc2 = parc.sjoin(site_fon[['site_id','geom']],how='left')\
.replace({'site_id':{'_ZI|_ZO':''}},regex=True)\
.drop(columns=['index_right'])\
.drop_duplicates()\
.reset_index(drop=True)
# Drop manuel des doublons
parc_CRAS = ['381370000B0249','381370000B0241','381370000B0248','381370000B0250','381370000B0582']
parc_MONA = ['383100000A0693']
id_drop1 = parc2[parc2.par_id.isin(parc_CRAS) & (parc2.site_id != 'CRAS')].index
id_drop2 = parc2[parc2.par_id.isin(parc_MONA) & (parc2.site_id != 'MONA')].index
lst_id_drop = [*id_drop1,*id_drop2]
parc2.drop(lst_id_drop,inplace=True)
parc2.loc[parc2.par_id == '381370000B0566','site_id'] = 'CRAS'
# Jointure type_zone
parc2['zi'] = parc2.intersects(site_zi.unary_union)
parc2['zo'] = parc2.intersects(site_zo.unary_union)
parc2.loc[parc2.zi,'type_zone'] = 'ZI'
parc2.loc[parc2.zo & (parc2.type_zone != 'ZI'),'type_zone'] = 'ZO'
parc2.drop(columns = ['zi','zo'],inplace=True)
# PARC = gpd.pd.concat([parc2,parc_hs2])
parc2['last_upd_parc'] = dt.now().date().isoformat()
# Envoie bd-cen-38.foncier.c_anim_foncier_v2
parc2.to_postgis(
name = 'c_anim_foncier_v2',
con = con_bdcen,
schema = 'foncier',
if_exists = 'replace',
index=False
)
INVALID_parc_hs.drop(columns='geom').to_sql(
name = 'c_anim_foncier_v2',
con = con_bdcen,
schema = 'foncier',
if_exists = 'append',
index=False
)
sql = """
ALTER TABLE foncier.c_anim_foncier_v2 ADD PRIMARY KEY (par_id);
GRANT ALL ON TABLE foncier.c_anim_foncier_v2 TO abavarot;
GRANT ALL ON TABLE foncier.c_anim_foncier_v2 TO grp_admin;
GRANT ALL ON TABLE foncier.c_anim_foncier_v2 TO cen_admin;
GRANT SELECT ON TABLE foncier.c_anim_foncier_v2 TO grp_consult;
"""
with con_bdcen.begin() as cnx:
cnx.execute(sql)
0_FONCIER/create_foreign_zh.py
+126
View File
@@ -0,0 +1,126 @@
from pycen import con_fon
foreign_server = """
CREATE SERVER fdw_azalee
FOREIGN DATA WRAPPER postgres_fdw
OPTIONS (dbname 'azalee', host '91.134.194.221', port '5432')
"""
user_mapping = """
CREATE USER MAPPING
FOR cen_admin
SERVER fdw_azalee
OPTIONS (user 'cen_admin', password '#CEN38@venir')
"""
foreign_table = """
-- DROP FOREIGN TABLE inventaires.inventaire_zh;
CREATE FOREIGN TABLE inventaires.inventaire_zh (
site_code varchar(10) NULL,
nom varchar NULL,
auteur_site varchar NULL,
auteur_geom varchar NULL,
auteur_last_maj varchar NULL,
date_site date NULL,
date_geom date NULL,
date_last_maj date NULL,
type_milieu varchar NULL,
type_site varchar NULL,
typo_sdage varchar NULL,
rmq_site text NULL,
rmq_fct_majeur text NULL,
rmq_interet_patri text NULL,
rmq_bilan_menace text NULL,
rmq_orient_act text NULL,
rmq_usage_process text NULL,
code_cb varchar NULL,
lib_cb text NULL,
activ_hum varchar NULL,
impact varchar NULL,
"position" varchar NULL,
rmq_activ_hum text NULL,
connexion varchar NULL,
subm_orig varchar NULL,
subm_freq varchar NULL,
subm_etend varchar NULL,
fct_bio varchar NULL,
fct_hydro varchar NULL,
int_patri varchar NULL,
"val_socioEco" varchar NULL,
crit_delim varchar NULL,
crit_def_esp varchar NULL,
entree_eau_reg varchar NULL,
entree_eau_perm varchar NULL,
entree_eau_topo varchar NULL,
sortie_eau_reg varchar NULL,
sortie_eau_perm varchar NULL,
sortie_eau_topo varchar NULL,
geom public.geometry(geometry, 2154) NULL
)
SERVER fdw_azalee
OPTIONS (schema_name 'zones_humides', table_name 'v_zoneshumides');
-- Permissions
ALTER TABLE inventaires.inventaire_zh OWNER TO cen_admin;
GRANT ALL ON TABLE inventaires.inventaire_zh TO cen_admin;
"""
view_v_zoneshumides = """
-- inventaires.v_zoneshumides source
CREATE OR REPLACE VIEW inventaires.v_zoneshumides
AS SELECT inventaire_zh.site_code,
inventaire_zh.nom,
inventaire_zh.auteur_site,
inventaire_zh.auteur_geom,
inventaire_zh.auteur_last_maj,
inventaire_zh.date_site,
inventaire_zh.date_geom,
inventaire_zh.date_last_maj,
inventaire_zh.type_milieu,
inventaire_zh.type_site,
inventaire_zh.typo_sdage,
inventaire_zh.rmq_site,
inventaire_zh.rmq_fct_majeur,
inventaire_zh.rmq_interet_patri,
inventaire_zh.rmq_bilan_menace,
inventaire_zh.rmq_orient_act,
inventaire_zh.rmq_usage_process,
inventaire_zh.code_cb,
inventaire_zh.lib_cb,
inventaire_zh.activ_hum,
inventaire_zh.impact,
inventaire_zh."position",
inventaire_zh.rmq_activ_hum,
CASE
WHEN inventaire_zh.rmq_activ_hum ~~ '%remblai%'::text THEN 't'::text
ELSE 'f'::text
END AS remblais,
inventaire_zh.connexion,
inventaire_zh.subm_orig,
inventaire_zh.subm_freq,
inventaire_zh.subm_etend,
inventaire_zh.fct_bio,
inventaire_zh.fct_hydro,
inventaire_zh.int_patri,
inventaire_zh."val_socioEco",
inventaire_zh.crit_delim,
inventaire_zh.crit_def_esp,
inventaire_zh.entree_eau_reg,
inventaire_zh.entree_eau_perm,
inventaire_zh.entree_eau_topo,
inventaire_zh.sortie_eau_reg,
inventaire_zh.sortie_eau_perm,
inventaire_zh.sortie_eau_topo,
inventaire_zh.geom
FROM inventaires.inventaire_zh;
-- Permissions
ALTER TABLE inventaires.v_zoneshumides OWNER TO cen_admin;
GRANT ALL ON TABLE inventaires.v_zoneshumides TO cen_admin;
GRANT SELECT ON TABLE inventaires.v_zoneshumides TO grp_sig;
GRANT SELECT ON TABLE inventaires.v_zoneshumides TO cen_user;
"""
with con_fon.begin() as cnx:
cnx.execute(foreign_server)
cnx.execute(user_mapping)
cnx.execute(foreign_table)
cnx.execute(view_v_zoneshumides)
0_FONCIER/cutfantoir_bydep.py
+28
View File
@@ -0,0 +1,28 @@
import pandas as pd
import csv
path = '/media/cgeier/SRV/FICHIERS/TRANSFERTS-EQUIPE/CG/FONCIER/CADASTRE/2023/Fichier national FANTOIR (situation avril 2023)/'
fant = path+'FANTOIR0423'
lst_dep = ['07','26','38','42']
if __name__ == '__main__':
df = pd.read_table(fant,chunksize=500000)
d07 = pd.DataFrame()
d26 = pd.DataFrame()
d38 = pd.DataFrame()
d42 = pd.DataFrame()
for d in df:
d.columns=['tmp']
d07 = pd.concat([d07,d[d.tmp.str.startswith('070')]])
d26 = pd.concat([d26,d[d.tmp.str.startswith('260')]])
d38 = pd.concat([d38,d[d.tmp.str.startswith('380')]])
d42 = pd.concat([d42,d[d.tmp.str.startswith('420')]])
d07.to_csv(path+'070.txt', header=None, index=None, quoting=csv.QUOTE_NONE)
d26.to_csv(path+'260.txt', header=None, index=None, quoting=csv.QUOTE_NONE)
d38.to_csv(path+'380.txt', header=None, index=None, quoting=csv.QUOTE_NONE)
d42.to_csv(path+'420.txt', header=None, index=None, quoting=csv.QUOTE_NONE)
0_FONCIER/delete_foncier_acte.py
+15
View File
@@ -0,0 +1,15 @@
from pycen import con_fon
id_acte = 'MU_CHAR_ZI2'
if id_acte.startswith('MF'):
lst_tab = ['r_mfstatut','r_frais_mf','r_cad_site_mf','mf_acquisitions']
key = 'mf_id'
elif id_acte.startswith('MU'):
lst_tab = ['r_mustatut','r_frais_mu','r_cad_site_mu','mu_conventions']
key = 'mu_id'
sql = "DELETE FROM foncier.{tab} WHERE {key}='{acte_id}';"
for t in lst_tab:
with con_fon.begin() as cnx:
cnx.execute(sql.format(tab=t,key=key,acte_id=id_acte))
0_FONCIER/extract_cadastre.py
+102
View File
@@ -0,0 +1,102 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Nom : : extract_cadastre.py
#Description : Extraction du cadastre en fonction du des coordonnées NO,NE,SO,SE
# ou d'un polygon (.shp).
#Copyright : 2021, CEN38
#Auteur : Colas Geier
#Version : 1.0
import pandas as pd
import geopandas as gpd
from sqlalchemy import create_engine
from sqlalchemy.engine import URL
from geoalchemy2 import Geometry
# from shapely.geometry.multipolygon import MultiPolygon
from shapely.geometry.polygon import Polygon
from pyproj import Transformer
from pycen import con_fon as con
# import contextily as ctx
# import matplotlib.pyplot as plt
def geom_map(geom):
map1 = ctx.providers.OpenTopoMap
map2 = ctx.providers.Esri.WorldImagery
maps = geom.to_crs(epsg=3857).copy()
ax = maps.plot(alpha=0.5, edgecolor='k')
ctx.add_basemap(
ax,
attribution_size=5,
reset_extent=False,
source=map1,
zoom='auto'
# zoom=14
)
ax.set_axis_off()
output = '/home/colas/Documents/tmp/AUDE/cadastre_for_Aude.gpkg'
# Shape : chemin/nom.shp -- None
shp = None
# Coordonnées [[NO],[NE],[SE],[SO]] -- None
lat_point_list = [45.414249,45.414611,45.407813,45.407282] #,45.414249]
lon_point_list = [ 5.472428, 5.486879, 5.487019, 5.472701] #, 5.472428]
# bdd
bdd = True
code_site = 'CRAS'
type_zone = None
con_open = con.connect()
# Shape : chemin/nom.shp -- None
# shp = None
# Coordonnées [[NO],[NE],[SE],[SO]] -- None
# lat_point_list = []
# lon_point_list = []
if shp:
print('IMPORT du mask : %s'%shp)
polygon = gpd.read_file(shp)
if polygon.crs.srs != crs.lower():
polygon.to_crs(epsg, inplace=True)
elif bdd:
from pycen import con_bdcen
gdf = gpd.read_postgis("SELECT * FROM sites.c_sites_zonages WHERE code_site = '%s' --and type_zonage = '%s'"%(code_site,type_zone),con_bdcen)
polygon = gdf[['geom']]
polygon.rename_geometry('geometry',inplace=True)
else:
if epsg != 2154:
from pyproj import Transformer
transformer = Transformer.from_crs(crs, "EPSG:2154", always_xy=True)
lon_point_list,lat_point_list = transformer.transform(lon_point_list,lat_point_list)
epsg = 2154
crs = 'EPSG:%s'%epsg
polygon_geom = Polygon(zip(lon_point_list, lat_point_list))
polygon = gpd.GeoDataFrame(index=[0], crs=crs, geometry=[polygon_geom])
sql = '''SELECT t1.par_id, t1.geom, t1.codcom, t4.libelle vl,
t1.ccopre, t1.ccosec, t1.dnupla, t1.dparpi, t5.typprop_lib,t5.typprop, t1.ccocomm,
t1.ccoprem, t1.ccosecm, t1.dnuplam, t1.type,
t8.ccodem,
--substring(t8.dnupro from 3) dnupro,
--substring(t9.dnuper from 3) dnuper,
t9.ddenom, t9.jdatnss, t9.dldnss, t9.dsglpm, t9.dlign3, t9.dlign4,
t9.dlign5, t9.dlign6, t9.dnatpr, t9.gtoper, t9.ccogrm
FROM cadastre.parcelles_38 t1
LEFT JOIN cadastre.vl_38 t4 ON (t1.vl_id = t4.vl_id)
LEFT JOIN cadastre.d_typprop t5 ON (t1.typprop_id = t5.typprop_id)
LEFT JOIN cadastre.lots_38 t2 ON (t1.par_id = t2.par_id)
LEFT JOIN cadastre.lots_natcult_38 t3 ON (t2.lot_id = t3.lot_id)
LEFT JOIN cadastre.cadastre_38 t6 ON (t2.lot_id = t6.lot_id)
LEFT JOIN cadastre.cptprop_38 t7 ON (t6.dnupro = t7.dnupro)
LEFT JOIN cadastre.r_prop_cptprop_38 t8 ON (t7.dnupro = t8.dnupro)
LEFT JOIN cadastre.proprios_38 t9 ON (t8.dnuper = t9.dnuper)
WHERE ST_Intersects (t1.geom, 'SRID={epsg};{poly}');'''.format(epsg=epsg,poly=polygon.geometry[0])
df = gpd.read_postgis(
sql=sql,
con=con,
)
for col in df.columns:
if (df[col].isna()).all():
del df[col]
df.to_file(output,driver='GPKG')
0_FONCIER/foncier_call_fonctions.py
Executable
+137
View File
@@ -0,0 +1,137 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Nom : : foncier_insert_table.py
#Description : Correspondance entre les sites et les parcelles cadastrales.
#Copyright : 2021, CEN38
#Auteur : Colas Geier
#Version : 1.0
import pandas as pd
import geopandas as gpd
from sqlalchemy import create_engine #, func
# from sqlalchemy.orm import sessionmaker
import psycopg2
import datetime as dt
import sys
import gc
from pycen import con_fon
# Parametres généraux
epsg = '2154'
crs = 'EPSG:%s'%epsg
chunk = None
pci_annee = '2021'
matrice_annee = '2021'
start_time = dt.datetime.today()
def time_exec (init_time):
time = dt.datetime.today() - init_time
return str(time)
def get_data(table,schema,engine=con_fon,chunk=chunk):
sql = "SELECT * FROM {0}.{1}".format(schema, table)
tmp = gpd.read_postgis(
sql = sql,
con = con_fon,
geom_col = 'geom',
crs = crs,
chunksize = chunk, )
# Mise en forme des données
# start_time = dt.datetime.today()
if chunk:
df = pd.concat(tmp, ignore_index=True)
else:
df = tmp.copy()
return df
if __name__ == "__main__":
###############
# Get sites
tab_site = 'sites'
sch_site = 'sites'
tab_fon = 'cadastre_site'
sch_fon = 'foncier'
print('''IMPORT data from schema: '%s' , table : '%s' '''%(sch_site,tab_site))
# sql = "SELECT * FROM {0}.{1}".format(sch_site, tab_site)
sql = '''
SELECT * FROM {0}.{1}
WHERE site_id = 'CRAS'
--WHERE site_id NOT IN (SELECT DISTINCT site_id FROM {2}.{3})
'''.format(sch_site, tab_site, sch_fon, tab_fon)
site = gpd.read_postgis(
# table_name = table_in,
sql = sql,
con = con_fon,
geom_col = 'geom',
# schema = schema_in,
crs = crs,
chunksize = chunk, )
# union_site = gpd.GeoSeries(site.geom.cascaded_union)
if site.empty:
print('Pas de nouveaux sites à lier au cadastre ====> EXIT')
sys.exit()
#################
# Get parcelles
tab_parc = 'cadastre'
sch_parc = 'parcelles'
print('''IMPORT data from schema: '%s' , table : '%s' '''%(sch_parc,tab_parc))
sql = """SELECT * FROM {sch}.{tab} WHERE ST_Intersects (geom, 'SRID={epsg};{poly}')
AND par_id NOT IN (SELECT par_id FROM {sch}.parcelles_cen)""".format(
sch=tab_parc, tab=sch_parc, epsg=epsg, poly=site.unary_union
)
parc = gpd.read_postgis(
sql = sql,
con = con_fon,
geom_col = 'geom',
crs = crs,
chunksize = chunk, )
parc_cent = parc.copy()
# parc_cent.geom = parc_cent.representative_point()
# parc_cent.geom = parc_cent.centroid
res = gpd.sjoin(site, parc_cent, predicate='intersects')
print('RUN fonction "import_parcelles_cen" pour {} lignes '.format(res.shape[0]))
# lst_site = res.site_id.unique()
start_time = dt.datetime.today()
res['sql'] = "SELECT cadastre.import_parcelles_cen('"+res.par_id+"','"+res.site_id+"',"+str(start_time.year)+");"
with con_fon.begin() as cnx:
res['sql'].map(lambda x: cnx.execute(x))
###############
# Get cptprop
lst_parid = "','".join(res.par_id.unique())
sql = '''
SELECT * FROM cadastre.cadastre
JOIN cadastre.lots USING (lot_id)
WHERE lots.par_id IN ('{lst_parid}')'''.format(lst_parid=lst_parid)
cptprop = pd.read_sql(
sql = sql,
con = con_fon,)
cptprop.drop_duplicates('dnupro', inplace=True)
start_time = dt.datetime.today()
print('RUN fonction "import_cptprop_cen" pour {} lignes '.format(cptprop.shape[0]))
cptprop['sql'] = "SELECT cadastre.import_cptprop_cen('"+cptprop.dnupro+"',"+str(start_time.year)+");"
with con_fon.begin() as cnx:
cptprop['sql'].map(lambda x: cnx.execute(x))
time_exec(start_time)
print('END fonction : import_cptprop_cen .......... %s'%time_exec(start_time))
site_id = 'GDMA'
with con_fon.begin() as cnx:
cnx.execute("SELECT admin_sig.refresh_mview_foncier('{site_id}')"\
.format(
site_id=site_id
)
)
0_FONCIER/foncier_insert_administratif.py
+437
View File
@@ -0,0 +1,437 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Nom : : foncier_insert_table.py
#Description : Insertion/MAJ des données administratives et territoriales à la base <foncier> lors de sa création.
#Copyright : 2021, CEN38
#Auteur : Colas Geier
#Version : 1.0
import pandas as pd
import geopandas as gpd
from sqlalchemy import create_engine
from geoalchemy2 import Geometry
from pycen import bdd
from shapely.geometry.multipolygon import MultiPolygon
from pydate import cdate
import sys
import os
####################################
####################################
####################################
# PARAMETRES
# Liste des tables à mettre à jour. ATTENTION : le respect des nomenclatures est importante
# Liste dispo : ['com', 'dpt', 'ter', 'histo_com', 'ter_com'] Cette liste doit être identique aux index de l'object : Table
# Correspondance tables BD_FONCIER : ['communes', 'departements', 'territoires', 'r_histo_com', 'r_ter_com']
run = ['com', 'dpt', 'ter', 'histo_com', 'ter_com']
user = 'colas_geier'
pwd = 'adm1n*fOncier'
# Parametres bdd
# user = 'postgres'
# pwd = 'tutu'
# adr = '192.168.60.9'
adr = '91.134.194.221'
port = '5432'
base = 'bd_cen'
# Connexion bdd
# bd = bdd.CEN(
# user = user,
# pwd = pwd,
# adr = adr,
# base = base
# schema = schema
# )
crs = 'EPSG:2154'
con = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user,pwd,adr,port,base), echo=False)
con_open = con.connect()
# create_engine('postgresql+psycopg2://cen_admin:#CEN38@venir@172.17.0.2:5432/bd_cen', echo=True)
# PATH
PATHIN = '/home/colas/'
LIST_DEPT = ['07', '26', '42', '38']
#
MAIN_FOLDER = 'Documents'
SUB_FOLDERS = os.path.join('5_BDD','1_QGIS')
FILE_COM = ['COMMUNE.shp']
FILE_DPT = ['DEPARTEMENT.shp']
FILE_TER = [
'EPCI.shp',
'PARC_OU_RESERVE.shp',
'BASSIN_VERSANT_TOPOGRAPHIQUE.shp',
'Contours_GRPT_AURA.shp' ]
# Dict table
IN_COM = [{
'id': 'id',
'insee_com': 'code_insee',
'nom': 'nom',
None: 'prec_plani',
None: 'statut',
None: 'canton',
'insee_arr': 'arrondisst',
'insee_dep': 'depart',
'insee_reg': 'region',
'population': 'popul',
None: 'multican',
'actif': 'actif',
None: 'epfl',
# 'geometry': 'geom',
}]
IN_DPT = [{
'id': 'id',
'nom': 'nom',
'insee_dep': 'insee_dep',
'insee_reg': 'insee_reg',
'date_creat': 'date_creat',
'date_maj': 'date_maj',
'actif': 'actif',
# 'geometry': 'geom',
}]
IN_TER = [{ # DICT epci
'': 'territoire_id',
'code_siren': 'siren',
'nom': 'territoire_lib',
'': 'territoire_sigle',
'nature': 'typterritoire_id',
'': 'administratif',
'': 'prefixe',
},{ # DICT parc_ou_reserve
'': 'territoire_id',
'id': 'siren', # absence de code siren ==> récup des 10 derniers charactères du champs ID
'toponyme': 'territoire_lib',
'': 'territoire_sigle',
'nature': 'typterritoire_id',
'': 'administratif',
'': 'prefixe',
},{ # DICT bassin_versant
'': 'territoire_id',
'code_hydro': 'siren',
'toponyme': 'territoire_lib',
'': 'territoire_sigle',
'id': 'typterritoire_id',
'': 'administratif',
'': 'prefixe',
},{ # DICT epci
'': 'territoire_id',
'sirengrpt': 'siren',
'grpt': 'territoire_lib',
'': 'territoire_sigle',
'nature': 'typterritoire_id',
'': 'administratif',
'': 'prefixe',
},]
# liste des couches administratives listé dans PATH_TER
administratif = ['epci']
# DICT typterritoire_id lorsqu'une correspondance
# n'est pas possible avec un champs de la table attributaire.
# Relation typterritoire_id / ID (de la table attributaire).
typterid = {
'bassvers' : 'bv'
}
mask_path = 'Documents/5_BDD/1_QGIS/'
mask_file = 'mask_parcelles_cadastre.shp'
####################################
####################################
####################################
# FONCTIONS
def join_typterritoire(df, join):
tab = join['table']
sch = join['schema']
ind = join['id']
on = join['on']
tmp = pd.read_sql_table(
table_name = tab,
con = con,
schema = sch,
index_col = ind,
)
df[on['x']] = df[on['x']].str.lower()
df[on['x']] = df[on['x']].replace(tmp[on['y']].str.lower().to_list(),tmp.index.to_list())
df = df[ df[on['x']].isin(tmp.index.to_list()) ]
return df
def find_files(File, main_path='Documents', sub_path=None):
'''
@File : list
@main_path : str
@sub_path : str
'''
sub = ''
if sub_path:
sub = sub_path
path = os.path.join(main_path, sub)
list_path = []
for F in File :
matches = [str(path) for path in Path(path).rglob(F)]
list_path += matches
return list_path
####################################
####################################
####################################
# MAIN
if os.path.exists(PATHIN + mask_path + mask_file):
mask = gpd.read_file(PATHIN + mask_path + mask_file, crs=crs)
else :
sys.exit()
Table = {
'dpt': {'schema':'administratif', 'table':'departements', 'file': FILE_DPT, 'geom': True, 'dict':IN_DPT},
'com' : {'schema':'administratif', 'table':'communes', 'file': FILE_COM, 'geom': True, 'dict':IN_COM},
# 'histo_com': {'schema':'administratif', 'table':'r_histo_com', 'file':None, 'geom': False, 'dict':None},
'ter': {'schema':'territoires', 'table':'territoires', 'file': FILE_TER, 'geom': True, 'dict':IN_TER, 'join': {
'schema':'territoires', 'table': 'd_typterritoire', 'id': 'typterritoire_id', 'on': {'x': 'typterritoire_id', 'y': 'typterritoire_lib'}}},
'ter_com': {'schema':'territoires', 'table':'r_ter_com', 'file':None, 'geom': False, 'dict':None}
}
for d, dep in enumerate(LIST_DEPT):
PATH = 'Documents/5_BDD/1_QGIS/202103_BDTOPO/bdtopo_dep{0}_202103_shapefile/BDT_3-0_SHP_LAMB93_D0{0}-ED2021-03-15/'.format(dep)
PATH_COM = [PATH + 'ADMINISTRATIF/COMMUNE.shp']
PATH_DPT = [PATH + 'ADMINISTRATIF/DEPARTEMENT.shp']
PATH_TER = [
PATH + 'ADMINISTRATIF/EPCI.shp',
PATH + 'ZONES_REGLEMENTEES/PARC_OU_RESERVE.shp',
PATH + 'HYDROGRAPHIE/BASSIN_VERSANT_TOPOGRAPHIQUE.shp',
'' ]
Table = {
'dpt': {'schema':'administratif', 'table':'departements', 'file': [PATHIN + path for path in PATH_DPT], 'geom': True, 'dict':IN_DPT},
'com' : {'schema':'administratif', 'table':'communes', 'file': [PATHIN + path for path in PATH_COM], 'geom': True, 'dict':IN_COM},
# 'histo_com': {'schema':'administratif', 'table':'r_histo_com', 'file':None, 'geom': False, 'dict':None},
'ter': {'schema':'territoires', 'table':'territoires', 'file': [PATHIN + path for path in PATH_TER], 'geom': True, 'dict':IN_TER, 'join': {
'schema':'territoires', 'table': 'd_typterritoire', 'id': 'typterritoire_id', 'on': {'x': 'typterritoire_id', 'y': 'typterritoire_lib'}}},
'ter_com': {'schema':'territoires', 'table':'r_ter_com', 'file':None, 'geom': False, 'dict':None}
}
not_run = [k for k in Table.keys() if k not in run]
for r in not_run:
del Table[r]
for tab in Table:
Table['file'] = find_files(File= Table['file'], sub_path=SUB_FOLDERS, main_path=MAIN_FOLDER)
if d == 0:
for tab in reversed(Table):
# continue
sql = "TRUNCATE TABLE {0}.{1} CASCADE".format(Table[tab]['schema'], Table[tab]['table'])
print(sql)
con_open.execute(sql)
no_r_tab = lambda x: x not in ['ter_com']
for key in filter(no_r_tab, Table.keys()):
# Test existance de la table en bdd
lst_tab = con.dialect.get_table_names(con, schema=Table[key]['schema'])
test = Table[key]['table'] in lst_tab
# Si la table existe
if test:
DICT = Table[key]['dict']
# Test présence d'un champ 'geom' ou 'geometry' dans la table d'export
geom = False
col_tab = con.dialect.get_columns(con, Table[key]['table'], schema=Table[key]['schema'])
for o, obj in enumerate(col_tab):
if 'geom' in obj['name']:
geom = True
geom_name = obj['name']
geom_type = obj['type'].geometry_type
if DICT:
for D, tmp in enumerate(DICT):
DICT[D]['geometry'] = geom_name
# Suppression des champs non utiles
if DICT:
for D, tmp in enumerate(DICT):
if DICT[D] and None in DICT[D].keys():
del DICT[D][None]
if Table[key]['file']:
for f, i_file in enumerate(Table[key]['file']):
# Si présence d'une géometrie dans la table à insérer
if geom:
# if Table[key]['geom']:
# Read new table
print('IMPORT shape for table {0}'.format(Table[key]['table']))
df = gpd.read_file(filename=i_file)
df = gpd.sjoin(df, mask, how='inner', op='intersects', rsuffix='right')
del_cols = [col for col in tutu.columns if col.endswith('right')] + ['FID']
df.drop(columns=del_cols, inplace=True)
df['actif'] = True
# if 'ID' in df.columns:
# df.set_index('ID', inplace=True)
# typ_geom_out = con.dialect.get_columns(con, Table[key]['table'], schema=Table[key]['schema'])
# Harmonisation des géometries
# Transformation des géometries POLYGON ==> MULTIPOLIGON
geom_df = df.geometry.geom_type.unique().tolist()
geom_df = [x.upper() for x in geom_df]
if [geom_type] != geom_df:
if geom_type == 'MULTIPOLYGON' and 'POLYGON' in geom_df:
print('CORRECTION des géometries POLYGON ==> MULTIPOLYGON')
lst = []
for o, obj in enumerate(df.geometry):
if obj.geom_type == 'Polygon':
obj = MultiPolygon([obj])
lst.append(obj)
df['geometry'] = lst
# elif geom_type == 'POLYGON' and 'MULTIPOLYGON' in geom_df:
# df[df.geom.geom_type == 'MultiPolygon']
# pass
else:
print('ERROR : conflit entre la géometrie du df {0} et de la table postgis {1}'.format(geom_df,geom_type))
# Conservation des lignes appartenant au département
print('CONSERVATION des données départementales')
df.columns = df.columns.str.lower()
if 'insee_dep' in df.columns:
df = df.loc[df.insee_dep == dep]
# Formatage des champs pour insertion
print('FORMATAGE des données')
df.rename(columns=DICT[f], inplace=True)
rm_col = df.columns.difference(DICT[f].values())
df.drop(columns=rm_col, inplace=True)
# break
# Identification du champs 'administratif' pour la table territoire
couche = i_file.split('/')
couche = couche[len(couche)-1]
couche = couche.split('.')[0].lower()
if 'ter' == key and couche in administratif:
df['administratif'] = True
elif 'ter' == key and couche not in administratif:
df['administratif'] = False
df['siren'] = [siren[-10:] for siren in df['siren']]
if 'typterritoire_id' in DICT[f].values():
key_typterr = [k for (k, v) in DICT[f].items() if v == 'typterritoire_id'][0]
if 'join' in Table[key].keys() and key_typterr != 'id':
df = join_typterritoire(df, Table[key]['join'])
if key == 'ter' and key_typterr == 'id':
# df['typterritoire_id'] = df.index.to_list()
df['typterritoire_id'] = [typter[:8] for typter in df['typterritoire_id']]
df['typterritoire_id'] = df['typterritoire_id'].str.lower()
df['typterritoire_id'] = df['typterritoire_id'].replace(typterid)
# df[on['x']].replace(tmp[on['y']].str.lower().to_list(),tmp.index.to_list())
# Si présence d'une géometrie dans la table à insérer
if geom:
if not isinstance(df, gpd.GeoDataFrame):
df = df.set_geometry('geom', drop=True, crs=crs)
df.rename(columns={'geometry': 'geom'}, inplace=True)
# if Table[key]['geom']:
df.to_postgis(
name = Table[key]['table'],
con = con,
schema = Table[key]['schema'],
index = False,
if_exists = 'append',
geom_col = geom_name,
# dtype={'geom': Geometry(geometry_type='MULTIPOLYGON', srid=df.crs.to_epsg())}
)
print('''INSERT TABLE OK for DEPT {}
'''.format(dep))
else:
None
# Si la table existe pas
else:
print('ERROR : La table {0} n\'existe pas dans le schéma {1} !'.format(
Table[key]['table'].upper(),
Table[key]['schema'].upper()
))
if 'ter_com' in Table.keys() and d == 0:
tab = Table['ter_com']
print('IMPORT tables for table {0}'.format(tab['table']))
ter_sql = 'SELECT * FROM {sch}.{tab}'.format(sch='territoires', tab='territoires' )
ter = gpd.read_postgis(
sql = ter_sql,
con = con,
geom_col = 'geom',
crs = crs,
)
com_sql = 'SELECT * FROM {sch}.{tab} WHERE actif = true'.format(sch='administratif', tab='communes' )
com = gpd.read_postgis(
sql = com_sql,
con = con,
geom_col = 'geom',
crs = crs,
)
col_id = ['territoire_id', 'code_insee', 'geom']
for df in [ter, com]:
rm_col = [ col for col in df.columns[~df.columns.isin(col_id)] ]
df.drop(columns=rm_col, inplace=True)
print('JOIN tables "territoires" & "communes"')
df = gpd.sjoin(ter, com, op='intersects')
rm_col = [ col for col in df.columns[~df.columns.isin(col_id)] ]
rm_col.append('geom')
df.drop(columns=rm_col, inplace=True)
df = pd.DataFrame(df)
r_sql = 'SELECT code_insee, territoire_id FROM {sch}.{tab}'.format(sch=tab['schema'], tab=tab['table'] )
r_tab = pd.read_sql(
sql = r_sql,
con = con,
)
if not r_tab.empty:
print('DROP lignes présentes dans la table {}'.format(tab['table']))
df = pd.concat([df,r_tab]).drop_duplicates(keep=False)
if not df.empty:
users = pd.read_sql_table(
table_name = 'utilisateurs',
con = con,
schema = 'admin_sig'
)
date_now = cdate.today()
df['actif'] = True
df['date_maj'] = date_now
df['utilisateur_id'] = users[users.utilisateur_id == user].iloc[0]['individu_id']
df.to_sql(
name = tab['table'],
con = con,
schema = tab['schema'],
index = False,
if_exists = 'append'
)
else:
print('''
TOUTES les relations "communes" / "territoires" existent déjà !
''')
# gdf.set_index('id', drop=True, inplace=True)
# for key in Table.keys():
# # schema = Table[key]['schema']
# # table = Table[key]['table']
# # query = 'SELECT * FROM {0}.{1}'.format(schema,table)
# # gdf = gpd.read_postgis(sql=query, con=con)
# df = bd.get_table(
# schema=Table[key]['schema'],
# table=Table[key]['table'])
# print(df)
0_FONCIER/foncier_insert_administratif_V2.py
Executable
+573
View File
@@ -0,0 +1,573 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Nom : : foncier_insert_table.py
#Description : Insertion/MAJ des données administratives et territoriales à la base <foncier> lors de sa création.
#Copyright : Mai 2021, CEN38
#Auteur : Colas Geier
#Version : 2.0
import pandas as pd
import geopandas as gpd
from sqlalchemy import create_engine, text
from geoalchemy2 import Geometry
from shapely.geometry.multipolygon import MultiPolygon
from pydate import cdate
from pathlib import Path
from pycen import con_fon as con
import sys
import os
# from pycen import bdd
# import psycopg2
####################################
####################################
####################################
# PARAMETRES
# Liste des tables à mettre à jour. ATTENTION : le respect des nomenclatures est importante
# Liste dispo : ['com', 'dpt', 'ter', 'ter_com'] Cette liste doit être identique aux index de l'object : Table
# Correspondance tables BD_FONCIER : ['communes', 'departements', 'territoires', 'r_histo_com', 'r_ter_com']
run = ['com']
# INSERT, TRUNCATE, UPDATE
action = 'UPDATE'
# Parametres bdd
# user = 'remi_clement'
# pwd = 'adm1n*CEN'
# adr = '192.168.60.9'
user = 'colas_geier'
pwd = 'adm1n*fOncier'
adr = '91.134.194.221'
port = '5432'
base = 'bd_cen'
epsg = '2154'
crs = 'EPSG:%s'%epsg
# PATH
INDEX_FOLDER = '/home/colas' # "/home/USER" (Unix), "C:" ou autre (Windows)
MAIN_FOLDER = os.path.join(INDEX_FOLDER,'Documents') # Dossier principale
SUB_FOLDERS = os.path.join('5_BDD','1_QGIS') # Sous dossier (facultatif: os.path.join('subFolder1','subFolder2'), None si non précisé)
FILE_COM = ['COMMUNE.shp']
FILE_DPT = ['DEPARTEMENT.shp']
FILE_TER = [
'EPCI.shp',
'PARC_OU_RESERVE.shp',
'BASSIN_VERSANT_TOPOGRAPHIQUE.shp',
'Contours_GRPT_AURA.shp' ]
administratif = ['epci'] # liste des couches administratives listé dans PATH_TER
mask_Mfolder = os.path.join(INDEX_FOLDER,'Documents')
mask_Sfolder = os.path.join('5_BDD','1_QGIS')
mask_file = 'mask_parcelles_cadastre.shp'
# DICT FILE / TABLE
IN_COM = [{
'id': 'id',
'insee_com': 'code_insee',
'nom': 'nom',
None: 'prec_plani', # Plus fournis par la bd_topo
None: 'statut', # Plus fournis par la bd_topo
None: 'canton', # Plus fournis par la bd_topo
'insee_arr': 'arrondisst',
'insee_dep': 'depart',
'insee_reg': 'region',
'population': 'popul',
None: 'multican', # Plus fournis par la bd_topo
'actif': 'actif',
None: 'epfl', # Plus fournis par la bd_topo
}]
IN_DPT = [{
'id': 'id',
'nom': 'nom',
'insee_dep': 'insee_dep',
'insee_reg': 'insee_reg',
'date_creat': 'date_creat',
'date_maj': 'date_maj',
'actif': 'actif',
}]
IN_TER = [{ # DICT epci
'': 'territoire_id',
'code_siren': 'siren',
'nom': 'territoire_lib',
'': 'territoire_sigle',
'nature': 'typterritoire_id',
'': 'administratif',
'': 'prefixe',
},{ # DICT parc_ou_reserve
'': 'territoire_id',
'id': 'siren', # absence de code siren ==> récup des 10 derniers charactères du champs ID
'toponyme': 'territoire_lib',
'': 'territoire_sigle',
'nature': 'typterritoire_id',
'': 'administratif',
'': 'prefixe',
},{ # DICT bassin_versant
'': 'territoire_id',
'code_hydro': 'siren', # absence de code siren ==> récup des 10 derniers charactères du champs ID
'toponyme': 'territoire_lib',
'': 'territoire_sigle',
'id': 'typterritoire_id',
'': 'administratif',
'': 'prefixe',
},{ # DICT epci
'': 'territoire_id',
'sirengrpt': 'siren',
'grpt': 'territoire_lib',
'': 'territoire_sigle',
'nature': 'typterritoire_id',
'': 'administratif',
'': 'prefixe',
},]
# DICT typterritoire_id lorsqu'une correspondance
# n'est pas possible avec un champs de la table attributaire.
# Relation typterritoire_id / ID (de la table attributaire).
typterid = {
'bassvers' : 'bv'
}
Table = {
'dpt': {'schema':'administratif', 'table':'departements', 'name_file': FILE_DPT, 'geom': True, 'dict':IN_DPT, 'unique': 'insee_dep'},
'com' : {'schema':'administratif', 'table':'communes', 'name_file': FILE_COM, 'geom': True, 'dict':IN_COM, 'unique': 'code_insee'},
# 'histo_com': {'schema':'administratif', 'table':'r_histo_com', 'name_file':None, 'geom': False, 'dict':None},
'ter': {'schema':'territoires', 'table':'territoires', 'name_file': FILE_TER, 'geom': True, 'dict':IN_TER, 'unique': 'siren', 'join': {
'schema':'territoires', 'table': 'd_typterritoire', 'id': 'typterritoire_id', 'on': {'x': 'typterritoire_id', 'y': 'typterritoire_lib'}}},
'ter_com': {'schema':'territoires', 'table':'r_ter_com', 'name_file':None, 'geom': False, 'dict':None, 'unique': ['code_insee', 'territoire_id']}
}
####################################
####################################
####################################
# FONCTIONS
def join_typterritoire(df, join):
'''
@df : dataframe
@join : dict
'''
tab = join['table']
sch = join['schema']
ind = join['id']
on = join['on']
tmp = pd.read_sql_table(
table_name = tab,
con = con,
schema = sch,
index_col = ind,
)
df[on['x']] = df[on['x']].str.lower()
df[on['x']] = df[on['x']].replace(tmp[on['y']].str.lower().to_list(),tmp.index.to_list())
df = df[ df[on['x']].isin(tmp.index.to_list()) ]
return df
def find_files(File, main_path='Documents', sub_path=None):
'''
@File : list
@main_path : str
@sub_path : str
'''
sub = ''
if sub_path:
sub = sub_path
path = os.path.join(main_path, sub)
list_path = []
nb_path = []
for F in File :
matches = [str(path) for path in Path(path).rglob(F)]
list_path += matches
nb_path += str(len(matches))
return list_path, nb_path
def tab_has_data(con, schema, table):
'''
@con : connection sqlalchemy create_engine
@schema : str
@table : str
'''
has_sch = con.dialect.has_schema(con, schema=schema)
if has_sch :
has_table = con.dialect.has_table(con, table_name=table, schema=schema)
if has_table:
sql = 'SELECT * FROM {sch}.{tab} LIMIT 1'.format(sch=schema,tab=table)
df = pd.read_sql_query(
sql = sql,
con = con
)
return not df.empty
else:
return '''TABLE %s don't exist in SCHEMA %s''' %(table,schema)
else :
return '''SCHEMA %s don't exist'''%schema
def update_data(df, con, sch, tab, epsg=None):
columns = df.columns.to_list()
frame = df.copy()
frame.replace("'","''", regex=True, inplace=True)
pkey = con.dialect.get_pk_constraint(con, table_name=tab, schema=sch)['constrained_columns']
if 'geom' in columns or 'geometry' in columns:
if epsg or df.crs:
if not epsg:
epsg = df.crs.to_epsg()
name_geom = df.geometry.name
frame[name_geom] = 'SRID={epsg};'.format(epsg=epsg) + df[name_geom].map(str)
# else: return 'No crs define in update_data or in gdf'
for c, col in enumerate(columns):
if c == 0:
frame['insert'] = "('" + frame[col].map(str)
# break
else:
frame['insert'] = frame['insert'] + "','" + frame[col].map(str)
if c == len(columns)-1:
frame['insert'] = frame['insert'] + "')"
# if c == 0:
# frame['insert'] = '("' + frame[col].map(str)
# # break
# else:
# frame['insert'] = frame['insert'] + '","' + frame[col].map(str)
# if c == len(columns)-1:
# frame['insert'] = frame['insert'] + '")'
lst_cols = ', '.join(columns)
lst_vals = ','.join(frame['insert'])
lst_dupKey = ', '.join([col + '=EXCLUDED.' + col for col in columns])
lst_pkey = ','.join(pkey)
sql = '''INSERT INTO {sch}.{tab} ({lst_cols}) VALUES {lst_vals} ON CONFLICT ({lst_pkey}) DO UPDATE SET {lst_dupKey} ;'''.format(
sch=sch, tab=tab, lst_cols=lst_cols, lst_vals=lst_vals, lst_dupKey=lst_dupKey, lst_pkey=lst_pkey)
# sql = '''INSERT INTO {sch}.{tab} ({lst_cols})
# VALUES {lst_vals}
# ON CONFLICT DO NOTHING;
# '''.format(sch=sch, tab=tab, lst_cols=lst_cols, lst_vals=lst_vals)
try:
con.execute(sql)
# con.execute(text(sql))
print('''
Update OK !''')
except Exception as exept:
print(exept)
# con = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user,pwd,adr,port,base), echo=False)
con_open = con.connect()
# conn = psycopg2.connect(
# user=user,
# password=pwd,
# host=adr,
# port=port,
# database=base)
####################################
####################################
####################################
# MAIN
if __name__ == "__main__":
mask_f = find_files(File=[mask_file], sub_path=mask_Sfolder, main_path=mask_Mfolder)[0]
if mask_f:
mask = gpd.read_file(mask_f[0], crs=crs)
else :
sys.exit('NO MASK FILE')
not_run = [k for k in Table.keys() if k not in run]
for r in not_run:
del Table[r]
for tab in Table :
if Table[tab]['name_file']:
Table[tab]['file'], Table[tab]['nb_typ_file'] = find_files(File=Table[tab]['name_file'], sub_path=SUB_FOLDERS, main_path=MAIN_FOLDER)
d = []
for i, nb in enumerate(Table[tab]['nb_typ_file']):
d += [ Table[tab]['dict'][i] ] * int(nb)
Table[tab]['dict'] = d
else:
Table[tab]['file'] = None
if action == 'TRUNCATE':
for tab in reversed(Table):
# continue
sql = "TRUNCATE TABLE {0}.{1} CASCADE".format(Table[tab]['schema'], Table[tab]['table'])
print(sql)
con_open.execute(sql)
sys.exit('END TRUNCATE')
else:
# filter1 = lambda x: x not in ['ter_com']
# for key in filter(filter1, Table.keys()):
for key in Table.keys():
# Test existance de données dans la table en bdd
has_data = tab_has_data(con, schema=Table[key]['schema'], table=Table[key]['table'])
# Si la table existe
if type(has_data) == bool :
# récupération des données présentent en bdd
if type(Table[key]['unique']) == str:
Table[key]['unique'] = [ Table[key]['unique'] ]
df_exist = False
if has_data :
df_exist = pd.read_sql_table(
con = con,
schema = Table[key]['schema'],
table_name = Table[key]['table'],
columns = Table[key]['unique']
)
DICT = Table[key]['dict']
# Test présence d'un champ 'geom' ou 'geometry' dans la table d'export
# Si géometrie, ajout du champs au dictionnaire
geom = False
col_tab = con.dialect.get_columns(con, Table[key]['table'], schema=Table[key]['schema'])
for o, obj in enumerate(col_tab):
if 'geom' in obj['name']:
geom = True
geom_name = obj['name']
geom_type = obj['type'].geometry_type
if DICT:
for D, tmp in enumerate(DICT):
DICT[D]['geometry'] = geom_name
# Récupération des données existantes dans la base de données
# Suppression des champs non utiles
if DICT:
for D, tmp in enumerate(DICT):
if DICT[D] and None in DICT[D].keys():
del DICT[D][None]
if Table[key]['file']:
for f, i_file in enumerate(Table[key]['file']):
# Test existance de données dans la table en bdd
has_data = tab_has_data(con, schema=Table[key]['schema'], table=Table[key]['table'])
# Si la table existe
if type(has_data) == bool :
# récupération des données présentent en bdd
if type(Table[key]['unique']) == str:
Table[key]['unique'] = [ Table[key]['unique'] ]
df_exist = False
if has_data :
pkey = con.dialect.get_pk_constraint(con, table_name=Table[key]['table'], schema=Table[key]['schema'])['constrained_columns']
df_exist = pd.read_sql_table(
con = con,
schema = Table[key]['schema'],
table_name = Table[key]['table'],
columns = Table[key]['unique'],
index_col = pkey
)
if df_exist.shape[1] == 0 :
df_exist[Table[key]['unique']] = df_exist.index
print('''
IMPORT {1} for table {0}'''.format(Table[key]['table'], i_file))
# Si présence d'une géometrie dans la table à insérer
if geom:
# if Table[key]['geom']:
# Read new table
df = gpd.read_file(filename=i_file)
if not df.crs:
df.set_crs(crs=crs, inplace=True)
df = gpd.sjoin(df, mask, how='inner', op='intersects', rsuffix='right')
del_cols = [col for col in df.columns if col.endswith('right')] + ['FID']
df.drop(columns=del_cols, inplace=True)
df['actif'] = True
geom_df = df.geometry.geom_type.unique().tolist()
geom_df = [x.upper() for x in geom_df]
if [geom_type] != geom_df:
if geom_type == 'MULTIPOLYGON' and 'POLYGON' in geom_df:
print('CORRECTION des géometries POLYGON ==> MULTIPOLYGON')
lst = []
for o, obj in enumerate(df.geometry):
if obj.geom_type == 'Polygon':
obj = MultiPolygon([obj])
lst.append(obj)
df['geometry'] = lst
# elif geom_type == 'POLYGON' and 'MULTIPOLYGON' in geom_df:
# df[df.geom.geom_type == 'MultiPolygon']
# pass
else:
print('ERROR : conflit entre la géometrie du df {0} et de la table postgis {1}'.format(geom_df,geom_type))
else:
# Si le fichier à importer ne possède pas de géometrie
print('NO geom !')
print('IMPORT data without geom : No-config !')
# Conservation des lignes appartenant au département
print('CONSERVATION des données départementales')
df.columns = df.columns.str.lower()
# if 'insee_dep' in df.columns:
# df = df.loc[df.insee_dep == dep]
# Formatage des champs pour insertion
print('FORMATAGE des données')
df.rename(columns=DICT[f], inplace=True)
rm_col = df.columns.difference(DICT[f].values())
df.drop(columns=rm_col, inplace=True)
# Identification du champs 'administratif' pour la table territoire
couche = i_file.split('/')
couche = couche[len(couche)-1]
couche = couche.split('.')[0].lower()
if 'ter' == key and couche in administratif:
df['administratif'] = True
elif 'ter' == key and couche not in administratif:
df['administratif'] = False
if df['siren'].dtypes == float:
df['siren'] = df['siren'].astype(int)
if df['siren'].dtypes == int:
df['siren'] = df['siren'].astype(str)
df['siren'] = [siren[-10:] for siren in df['siren']]
if 'typterritoire_id' in DICT[f].values():
key_typterr = [k for (k, v) in DICT[f].items() if v == 'typterritoire_id'][0]
if 'join' in Table[key].keys() and key_typterr != 'id':
df = join_typterritoire(df, Table[key]['join'])
if key == 'ter' and key_typterr == 'id':
df['typterritoire_id'] = [typter[:8] for typter in df['typterritoire_id']]
df['typterritoire_id'] = df['typterritoire_id'].str.lower()
df['typterritoire_id'] = df['typterritoire_id'].replace(typterid)
# if key == 'com':
# sys.exit()
# Suppression des lignes existantes en bdd
if action == 'INSERT' and has_data:
unique_cols = df_exist.columns.to_list()
for d in [df, df_exist]:
d['exist'] = ''
for col in unique_cols:
d['exist'] += d[col].astype(str)
df = df[~df['exist'].isin(df_exist['exist'])]
df.drop(columns='exist', inplace=True)
if action == 'UPDATE':
unique_cols = df_exist.columns.to_list()
for d in [df, df_exist]:
d['exist'] = ''
for col in unique_cols:
d['exist'] += d[col].astype(str)
df = df[df['exist'].isin(df_exist['exist'])].sort_values(unique_cols)
ind = df_exist[df_exist['exist'].isin(df['exist'])].sort_values(unique_cols).index
df.set_index(ind, inplace=True)
df.drop(columns='exist', inplace=True)
# Si présence d'une géometrie dans la table à insérer
if geom and not df.empty and action == 'INSERT' :
if not isinstance(df, gpd.GeoDataFrame):
df = df.set_geometry('geom', drop=False, crs=crs)
# df.rename(columns={'geometry': 'geom'}, inplace=True)
df.to_postgis(
name = Table[key]['table'],
con = con,
schema = Table[key]['schema'],
index = False,
if_exists = 'append',
geom_col = geom_name,
)
elif geom and df.empty and action == 'INSERT' :
print('NO NEWS data insert !')
# Si présence d'une géometrie dans la table à updater
elif geom and not df.empty and action == 'UPDATE' :
if not isinstance(df, gpd.GeoDataFrame):
df = df.set_geometry('geom', drop=False, crs=crs)
df.reset_index(inplace=True)
update_data(df, con, sch=Table[key]['schema'], tab=Table[key]['table'])
print('NO NEWS data update !')
sys.exit()
elif geom and df.empty and action == 'UPDATE' :
print('NO NEWS data update !')
else:
# Si les données à importer sont issues d'un fichier sans géometrie
print('FILE WITHOUT GEOM !')
else:
# Si il n'y a pas de données à importer
# Création des liens relationnelles Communes/Territoires.
print('NO IMPORT FILE !')
tab = Table[key]
print('IMPORT tables for table {0}'.format(tab['table']))
# SELECT data territoires
ter_sql = 'SELECT * FROM {sch}.{tab}'.format(sch='territoires', tab='territoires' )
ter = gpd.read_postgis(
sql = ter_sql,
con = con,
geom_col = 'geom',
crs = crs,
)
# SELECT DATA communes
com_sql = 'SELECT * FROM {sch}.{tab} WHERE actif = true'.format(sch='administratif', tab='communes' )
com = gpd.read_postgis(
sql = com_sql,
con = con,
geom_col = 'geom',
crs = crs,
)
# Conservation des données utiles
tab['unique'] += ['geom']
for df in [ter, com]:
rm_col = [ col for col in df.columns[~df.columns.isin(tab['unique'])] ]
df.drop(columns=rm_col, inplace=True)
# Jointure territoires VS communes
if not ter.empty and not com.empty:
print('JOIN tables "territoires" & "communes"')
df = gpd.sjoin(ter, com, op='intersects')
rm_col = [ col for col in df.columns[~df.columns.isin(tab['unique'])] ]
rm_col.append('geom')
df.drop(columns=rm_col, inplace=True)
df = pd.DataFrame(df)
# Récupération des données déjà présentent en bdd
# r_sql = 'SELECT code_insee, territoire_id FROM {sch}.{tab}'.format(sch=tab['schema'], tab=tab['table'] )
# r_tab = pd.read_sql(
# sql = r_sql,
# con = con,
# )
# if not r_tab.empty:
# print('DROP lignes présentes dans la table {}'.format(tab['table']))
# df = pd.concat([df,r_tab]).drop_duplicates(keep=False)
if action == 'INSERT' and has_data:
df = pd.concat([df,df_exist]).drop_duplicates(keep=False)
if action == 'INSERT' and not df.empty:
users = pd.read_sql_table(
table_name = 'utilisateurs',
con = con,
schema = 'admin_sig'
)
date_now = cdate.today()
df['actif'] = True
df['date_maj'] = date_now
df['utilisateur_id'] = users[users.utilisateur_id == user].iloc[0]['individu_id']
df.to_sql(
name = tab['table'],
con = con,
schema = tab['schema'],
index = False,
if_exists = 'append'
)
else:
print('''
TOUTES les relations "communes" / "territoires" existent déjà !
''')
elif ter.empty and not com.empty:
print('''
NO DATA dans la table "territoires" !
''')
elif not ter.empty and com.empty:
print('''
NO DATA dans la table "communes" !
''')
else:
print('''
NO DATA dans la table "communes" et dans la table "territoires" !
''')
sys.exit()
0_FONCIER/foncier_insert_cadastre.py
Executable
+770
View File
@@ -0,0 +1,770 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Nom : : foncier_insert_table.py
#Description : Insertion des données cadastrales à la base <foncier> après de sa création.
#Copyright : 2021, CEN38
#Auteur : Colas Geier
#Version : 1.0
import pandas as pd
# import numpy as np
from sqlalchemy import create_engine, text
from geoalchemy2 import Geometry
import gc
import sys
# import time
import datetime as dt
# from pycen import bdd
# from shapely.geometry.multipolygon import MultiPolygon
# Parametrage geopandas
import geopandas as gpd
import warnings; warnings.filterwarnings('ignore', 'GeoSeries.isna', UserWarning)
# import shapely
# shapely.speedups.disable()
# gpd.options.use_pygeos = True
# start_time = dt.datetime.today()
# tmp = dt.datetime.today() - start_time
check_duplicates = False
# Parametres bdd CADASTRE (in)
# Données de sortie du plugin qgis "Cadastre"
user_cad = 'postgres'
pwd_cad = 'foncier_test1'
adr_cad = '172.17.0.2'
port_cad = '5432'
base_cad = 'postgres'
schema_cad = '202007'
# Parametres bdd FONCIER (out)
user_fon = 'postgres'
pwd_fon = 'tutu'
adr_fon = '192.168.60.9'
port_fon = '5432'
base_fon = 'bd_cen'
schema_fon = 'cadastre'
# Correspondance entre les tables
crs = 'EPSG:2154'
dpt_nom_tab = '_73'
chunk = 100000
list_dep = ['07', '26', '42', '38']
FIND_DOUBLON = [{
'tab_in': 'proprietaire',
'on_col': ['ddenom', 'dprnlp', 'dldnss','jdatnss','ccogrm','dsglpm','dnatpr'] }
]
DICT_TAB = [{
'table_in' : 'proprietaire', # Table source qui provient de la sortie du plugin cadastre de qgis
'index_tab': 'proprietaire', # Pkey de la table source
'columns_in': ['ccodep', 'ccocom', 'dnupro',
'dnuper', 'ccoqua', 'ddenom', 'jdatnss', 'dldnss', 'dsglpm', 'dlign3', 'dlign4', 'dlign5', 'dlign6', 'dnatpr', 'gtoper', 'ccogrm'],
'table_out': [{
'name': 'cptprop{}'.format(dpt_nom_tab),
'geom': None,
'drop_escape': False, # Supprime les champs vides à l'intérieure des chaines de carractères
'columns_in': ['ccodep', 'ccocom', 'dnupro'], # Liste des columns à récupérer en entrée.
'columns_add': {'dnupro': ['ccodep', 'ccocom', 'dnupro']}, # Définition des champs composés devant être ajoutés
'unique': {'cols': ['dnupro'], 'keep': 'first'}, # Champs devant être uniques à l'intérieur de la table en sortie
'dict': None, # Dictionnaire pour renommer les champs {'ancien_nom1': 'nouveau_nom1', 'ancien_nom2': 'nouveau_nom2', ...}
'join': [{
'bdd': 'in', 'table': 'suf', 'on': ['ccodep', 'ccocom', 'dnupro'], 'type': 'concat',
'select_cols' : ['ccodep', 'ccocom', 'dnupro']},{
'bdd': 'in', 'table': 'lots', 'on': ['ccodep', 'ccocom', 'dnupro'], 'type': 'concat',
'select_cols' : ['ccodep', 'ccocom', 'dnuprol'],'dict': {'dnuprol': 'dnupro'}},{
'bdd': 'in', 'table': 'parcelle', 'on': ['ccodep', 'ccocom', 'dnupro'], 'type': 'concat',
'select_cols' : ['ccodep', 'ccocom', 'dnupro']},]
},{
'name': 'proprios{}'.format(dpt_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in': ['ccodep', 'dnuper', 'ccoqua', 'ddenom', 'jdatnss', 'dldnss', 'dsglpm', 'dlign3', 'dlign4', 'dlign5', 'dlign6', 'dnatpr', 'gtoper', 'ccogrm'],
'columns_add': {'dnuper': ['ccodep', 'dnuper']},
'unique': {'cols': ['dnuper'], 'keep': 'first'},
'dict': None,
'join': False
},{
'name': 'r_prop_cptprop{}'.format(dpt_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in': ['ccodep', 'dnuper', 'ccocom', 'dnupro', 'dnomlp', 'dprnlp', 'epxnee', 'dnomcp', 'dprncp', 'ccodro', 'ccodem'],
'columns_add': {
'dnuper': ['ccodep', 'dnuper'],
'dnupro': ['ccodep', 'ccocom', 'dnupro']},
'unique': {'cols': ['dnupro', 'dnuper'], 'keep': 'first'},
'dict': None,
'join': False
},]
},{
'table_in' : 'parcelle',
'index_tab': 'parcelle',
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'ccovoi', 'dparpi', 'dcntpa', 'ccocomm', 'ccoprem', 'ccosecm', 'dnuplam', 'dvoilib', 'type_filiation', 'dnupro'],
'table_out': [{
'name': 'vl{}'.format(dpt_nom_tab),
'geom': None,
'drop_escape': False,
'columns_in' : ['ccodep', 'ccocom', 'ccovoi', 'dvoilib'],
'columns_add': {
'vl_id': ['ccodep', 'ccocom', 'ccovoi'],
'geom': None},
'unique': {'cols': ['vl_id'], 'keep': 'first'},
'dict': {'dvoilib': 'libelle'},
'join': [{ # ERROR ! 2 dclssf pour 1 lot_id
'bdd': 'in', 'table': 'voie', 'on': ['ccodep', 'ccocom', 'ccovoi'], 'type': 'concat',
'select_cols' : ['ccodep', 'ccocom', 'codvoi', 'libvoi'],
'dict': {'libvoi': 'libelle', 'codvoi': 'ccovoi'},
}]
},{
'name': 'parcelles{}'.format(dpt_nom_tab),
'geom': {
'table_geom_in': 'geo_parcelle',
'index_geom': 'geo_parcelle'
},
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'ccovoi', 'dparpi', 'dcntpa', 'ccocomm', 'ccoprem', 'ccosecm', 'dnuplam', 'type_filiation'],
'columns_add': {
'par_id': ['ccodep', 'ccocom', 'ccopre','ccosec', 'dnupla'],
'codcom': ['ccodep', 'ccocom'],
'vl_id': ['ccodep', 'ccocom', 'ccovoi'],
'typprop_id': None },
'unique': False,
'dict': {'type_filiation': 'type'},
'join': False
},{
'name': 'lots{}'.format(dpt_nom_tab), # !!!!!! Ne trouve pas de parcelles sans lots (ex: 38357000AE0526)
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dcntpa'],
'columns_add': {
'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla'],
'par_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla'],
'dnulot': None, },
'unique': False,
'dict': {'dcntpa': 'dcntlo'},
'join': [{'bdd': 'out', 'table': 'parcelles{}'.format(dpt_nom_tab), 'on': ['par_id'], 'type': 'isin',
'select_cols' :['par_id'] }]
# },{
# 'name': 'cptprop{}'.format(dpt_nom_tab), # !!!!!! Ne trouve pas de parcelles sans lots (ex: 38357000AE0526)
# 'geom': None,
# 'drop_escape': True,
# 'columns_in' : ['ccodep', 'ccocom', 'dnupro'],
# 'columns_add': {
# 'dnupro': ['ccodep', 'ccocom', 'dnupro'],
# },
# 'unique': {'cols': ['dnupro'], 'keep': 'first'},
# 'dict': None,
# 'join': [{'bdd': 'out', 'table': 'cptprop{}'.format(dpt_nom_tab), 'on': ['dnupro'], 'type': 'notin',
# 'select_cols' :['dnupro'] }]
},]
# },{
# 'table_in' : 'suf',
# 'index_tab': 'suf',
# 'columns_in' : ['ccodep', 'ccocom', 'dnupro'],
# 'table_out': [{
# 'name': 'cptprop{}'.format(dpt_nom_tab), # !!!!!! Ne trouve pas de parcelles sans lots (ex: 38357000AE0526)
# 'geom': None,
# 'drop_escape': True,
# 'columns_in' : ['ccodep', 'ccocom', 'dnupro'],
# 'columns_add': {
# 'dnupro': ['ccodep', 'ccocom', 'dnupro'],
# },
# 'unique': {'cols': ['dnupro'], 'keep': 'first'},
# 'dict': None,
# 'join': [{'bdd': 'out', 'table': 'cptprop{}'.format(dpt_nom_tab), 'on': ['dnupro'], 'type': 'notin',
# 'select_cols' :['dnupro'] }]
# },]
},{
'table_in' : 'lots',
'index_tab': 'lots',
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnupdl', 'dcntlo', 'dnuprol'],
'table_out': [{
# 'name': 'cptprop{}'.format(dpt_nom_tab), # !!!!!! Ne trouve pas de parcelles sans lots (ex: 38357000AE0526)
# 'geom': None,
# 'drop_escape': True,
# 'columns_in' : ['ccodep', 'ccocom', 'dnuprol'],
# 'columns_add': {
# 'dnupro': ['ccodep', 'ccocom', 'dnuprol'],
# },
# 'unique': {'cols': ['dnupro'], 'keep': 'first'},
# 'dict': None,
# 'join': [{'bdd': 'out', 'table': 'cptprop{}'.format(dpt_nom_tab), 'on': ['dnupro'], 'type': 'notin',
# 'select_cols' :['dnupro'] }]
# },{
'name': 'lots{}'.format(dpt_nom_tab), # !!!!!! parcelles avec lots: existe par_id NOT IN parcelles_73
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnupdl', 'dcntlo'],
'columns_add': {
'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'],
'par_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla'],},
'unique': {'cols': ['lot_id'], 'keep': 'first'},
'dict': None,
'join': [{'bdd': 'out', 'table': 'parcelles{}'.format(dpt_nom_tab), 'on': ['par_id'], 'type': 'isin',
'select_cols' :['par_id'] }]
},{
'name': 'lots_natcult{}'.format(dpt_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'],
'columns_add': {
'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'],},
'unique': {'cols': ['lot_id'], 'keep': 'first'},
'dict': None,
'join': [{ # ERROR ! 2 dclssf pour 1 lot_id
'bdd': 'in', 'table': 'suf', 'on': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'], 'type': 'merge',
'select_cols' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot','dsgrpf','cnatsp','dclssf','ccosub','dcntsf'],
},{
'bdd': 'out', 'table': 'lots{}'.format(dpt_nom_tab), 'on': ['lot_id'], 'type': 'isin',
'select_cols' :['lot_id'] }]
},{
'name': 'cadastre{}'.format(dpt_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnuprol'],
'columns_add': {
'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'],
'dnupro': ['ccodep', 'ccocom', 'dnuprol'],},
'unique': {'cols': ['lot_id', 'dnupro'], 'keep': 'first'},
'dict': None,
'join': [{ # ERROR ! 2 dclssf pour 1 lot_id
'bdd': 'in', 'table': 'suf', 'on': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnuprol'], 'type': 'concat',
'select_cols' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnupro'], 'dict': {'dnupro': 'dnuprol'}
},{
'bdd': 'in', 'table': 'parcelle', 'on': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnuprol'], 'type': 'concat',
'select_cols' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnupro'], 'dict': {'dnupro': 'dnuprol'}
},{
'bdd': 'out', 'table': 'lots{}'.format(dpt_nom_tab), 'on': ['lot_id'], 'type': 'isin',
'select_cols' :['lot_id'] },{
'bdd': 'out', 'table': 'cptprop{}'.format(dpt_nom_tab), 'on': ['dnupro'], 'type': 'isin',
'select_cols' :['dnupro'] },]
},]
# },{
# 'table_in' : 'proprietaire', # Table source qui provient de la sortie du plugin cadastre de qgis
# 'index_tab': 'proprietaire', # Pkey de la table source
# 'columns_in': ['ccodep', 'dnuper', 'ccoqua', 'ddenom', 'jdatnss', 'dldnss', 'dsglpm', 'dlign3', 'dlign4', 'dlign5', 'dlign6', 'dnatpr', 'gtoper', 'ccogrm',
# 'ccocom', 'dnupro', 'dnomlp', 'dprnlp', 'epxnee', 'dnomcp', 'dprncp', 'ccodro', 'ccodem'],
# 'table_out': []
# },{
# 'table_in' : 'parcelle',
# 'index_tab': 'parcelle',
# 'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnupro'],
# 'table_out': [{
# 'name': 'cadastre{}'.format(dpt_nom_tab),
# 'geom': None,
# 'drop_escape': True,
# 'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnupro'],
# 'columns_add': {
# 'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla'],
# 'dnupro': ['ccodep', 'ccocom', 'dnupro'],},
# 'unique': {'cols': ['lot_id', 'dnupro'], 'keep': 'first'},
# 'dict': None,
# 'join': [{
# 'bdd': 'out', 'table': 'lots{}'.format(dpt_nom_tab), 'on': ['lot_id'], 'type': 'isin',
# 'select_cols' :['lot_id'], 'where': {'dnulot': None} },{
# 'bdd': 'out', 'table': 'cptprop{}'.format(dpt_nom_tab), 'on': ['dnupro'], 'type': 'isin',
# 'select_cols' :['dnupro'] },]
# },]
# },{
# 'table_in' : 'parcelle',
# 'index_tab': 'parcelle',
# 'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnupro'],
# 'table_out': [{
# 'name': 'cadastre{}'.format(dpt_nom_tab),
# 'geom': None,
# 'drop_escape': True,
# 'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnupro'],
# 'columns_add': {
# 'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla'],
# 'dnupro': ['ccodep', 'ccocom', 'dnupro'],},
# 'unique': {'cols': ['lot_id', 'dnupro'], 'keep': 'first'},
# 'dict': None,
# 'join': [{
# 'bdd': 'out', 'table': 'lots{}'.format(dpt_nom_tab), 'on': ['lot_id'], 'type': 'isin',
# 'select_cols' :['lot_id'], },{
# 'bdd': 'out', 'table': 'cptprop{}'.format(dpt_nom_tab), 'on': ['dnupro'], 'type': 'isin',
# 'select_cols' :['dnupro'] },]
# },]
}]
# # Connexion bdd
# bd_cad = bdd.CEN(
# user = user_cad,
# pwd = pwd_cad,
# adr = adr_cad,
# base = base_cad
# # schema = schema
# )
################################
########## Fonctions ##########
################################
start_time = dt.datetime.today()
def time_exec (init_time):
time = dt.datetime.today() - init_time
return str(time)
def replace_escape_by_0 (df):
# Remplacement des espaces dans les chaines de caractères par des 0
# if 'ccopre' in df.columns:
# df['ccopre'].replace([None, '', ' '], '000', inplace=True)
cols = ['ccopre', 'ccosec', 'dnupla', 'dparpi', 'dnuplam', 'dclssf', 'ccovoi']
for col in cols:
if col in df.columns:
df[col].replace([' '], '0', regex=True, inplace=True)
return df
def join_data (df, join, schema_in):
# Jointure des données avec une autre table
table = join['table']
bdd = join['bdd']
typ = join['type']
on = join['on']
if bdd == 'out':
con = engine_fon
sch = schema_fon
if bdd == 'in':
con = engine_cad
sch = schema_in
select_col = []
if 'select_cols' in join.keys():
select_col.extend(join['select_cols'])
if 'where' in join.keys():
select_col.extend(join['where'].keys())
tmp = pd.read_sql_table(
table_name = table,
con = con,
schema = sch,
columns = select_col
)
tmp = replace_escape_by_0(tmp)
if 'dict' in join.keys():
tmp.rename(columns=join['dict'], inplace=True)
if 'where' in join.keys():
where = join['where']
for key in where.keys():
tmp = tmp[tmp[key] == where[key] ]
if typ in ['isin', 'notin']:
# on = on[0]
for d in [df, tmp]:
d['on'] = ''
for col in on:
d['on'] += d[col].astype(str)
if typ == 'isin':
df = df[df['on'].isin(tmp['on'])]
if typ == 'notin':
df = df[~df['on'].isin(tmp['on'])]
df.drop(columns='on', inplace=True)
# if typ == 'notin':
# on = on[0]
# df = df[~df[on].isin(tmp[on])]
# df = pd.concat([df,tmp]).drop_duplicates(on, keep=False)
if typ == 'merge':
df = df.merge(tmp, on = on, how='left')
if typ == 'concat':
df = pd.concat([df,tmp], ignore_index=True).drop_duplicates()
return df
def get_geom_parcelle (df,get_geo,schema):
print('INIT import geodata ........... %s sec'%( time_exec(start_time) ))
# Définition des variables géometriques
ind_geo = get_geo['index_geom']
tab_geo = get_geo['table_geom_in']
sql = """select distinct on (t2.{0})
t2.{0},
t1.geom,
t1.supf::integer as dcntpa -- récupération de la contenance cadastrale associée car présence de géometrie non référencées dans la table "parcelles"
FROM "{1}".{2} t1
INNER JOIN (select distinct on ({0}) {0}, max(creat_date) creat_date, max(update_dat) update_dat FROM "{1}".{2} GROUP BY ({0})) t2
USING ({0}, creat_date, update_dat)""".format(ind_geo, schema, tab_geo)
tmp = gpd.read_postgis(
sql = sql,
con = engine_cad,
geom_col = 'geom',
crs = crs,
chunksize = chunk,
)
if chunk:
gdf = gpd.GeoDataFrame(pd.concat(tmp, ignore_index=True))
else:
gdf = tmp.copy()
# del tmp; gc.collect()
# gdf = tmp.copy()
del tmp
gdf.set_index(ind_geo, inplace=True)
gdf.index.name = ind_in
print('END import geodata ........... %s sec'%( time_exec(start_time) ))
print('INIT merge data - geodata ........... %s sec'%( time_exec(start_time) ))
if not gdf[gdf.dcntpa.isna()].empty:
gdf.dcntpa.fillna(0, inplace=True)
gdf['dcntpa'] = gdf['dcntpa'].astype(df.dtypes['dcntpa'].type)
# gdf = gdf.merge(df, on = [ind_in, 'dcntpa'], how='left')
tmp = gdf.merge(df, on = [ind_in, 'dcntpa'], how='right')
tmp = tmp.set_geometry('geom', drop=True, crs=crs)
tmp.rename(columns={'geometry': 'geom'}, inplace=True)
if tmp[tmp.geom.isna()].empty:
lst_ind_df = tmp[tmp.geom.isna()].index.tolist()
lst_ind_gdf = gdf.loc[gdf.index.isin(lst_ind_df)].index.tolist()
tmp.loc[tmp.index.isin(lst_ind_gdf), 'geom'] = gdf.loc[gdf.index.isin(lst_ind_gdf), 'geom']
del [gdf, df]
gdf = tmp.copy()
del tmp
export_data(gdf)
def export_data( df):
print('INIT export data TO {0}, {1} ........... {2} sec'.format(tab_out, df.shape[0], time_exec(start_time) ))
rang = [e for e in range(0, df.shape[0], chunk*5)]
for i, j in enumerate(rang):
if j == max(rang) :
jj = df.shape[0]
else:
jj = rang[i+1]
df_imp = df[j:jj].copy()
print('INIT export data TO {0} ..... {1}/{2} ...... {3} sec'.format(tab_out, jj, df.shape[0], time_exec(start_time) ))
if 'geom' in df.columns and not df[~df['geom'].isna()].empty :
df_imp = df_imp.set_geometry('geom', drop=True, crs=crs)
df_imp.rename(columns={'geometry': 'geom'}, inplace=True)
df_imp.to_postgis(
name = tab_out,
con = engine_fon,
schema = schema_fon,
index = False,
if_exists = 'append',
geom_col = 'geom',
chunksize = chunk,
)
else:
df_imp.to_sql(
name = tab_out,
con = engine_fon,
schema = schema_fon,
index = False,
if_exists = 'append',
chunksize = chunk,
method = 'multi',
)
print('END export data TO {0} ........... {1} sec'.format(tab_out, time_exec(start_time) ))
def optimize_data_frame(df):
columns = df.columns
for col in columns:
dtype = df[col].dtypes
# if dtype == 'int64' or dtype == 'int32':
len_col = len(df[col].unique())
if len_col <= df.shape[0]*0.8:
df[col] = df[col].astype('category')
return df
# Initiation des connexions bdd
engine_cad = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user_cad,pwd_cad,adr_cad,port_cad,base_cad), echo=False)
engine_fon = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user_fon,pwd_fon,adr_fon,port_fon,base_fon), echo=False)
con_cad = engine_cad.connect()
con_fon = engine_fon.connect()
################################
########## Main ##########
################################
if __name__ == "__main__":
################
# CORRECTION DUPLICATES TABLE_IN
if check_duplicates:
for DOUBLON in FIND_DOUBLON:
tab = DOUBLON['tab_in']
on_col = DOUBLON['on_col']
for col in on_col:
for dep in list_dep:
schema_in = dep + '_' + schema_cad
sql = '''
-- il existe des doublons en raison d'orthographes voisines :
-- recherche de ces doublons
SELECT DISTINCT '{0}' as insee_dep, dnuper, string_agg(DISTINCT {1},' / ') as orthographes_voisines
FROM "{2}".{3} GROUP BY dnuper HAVING count(DISTINCT {1}) > 1'''.format(dep, col, schema_in, tab)
df = pd.read_sql(
sql = sql,
con = engine_cad,
)
if df.empty:
print('No duplicate value dep {0} table {1} column {2} ====> next request'.format(dep, tab, col))
continue
for i, row in df.iterrows():
dnuper = row.dnuper
choix = row.orthographes_voisines.split(' / ')
choix = [i.strip() for i in choix]
Question = input("""Des orthographes voisines existent pour l'identifiant : {0}
dans la colonne : {1}.
Les valeurs voisines sont : {2}
Ecrire la mise à jour du champs {1} à enregistrer (c cancel) :""".format(dnuper,col, choix))
if Question.lower() == 'c' or Question.lower() == 'cancel':
continue
update = '''UPDATE "{0}".{1}
SET {2} = '{3}'
WHERE {2} like '{4}%'
AND dnuper = '{5}';'''.format(schema_in, tab, col, Question, "%' OR {} like '".format(col).join(map(str,choix)), dnuper)
try:
con_cad.execute(text(update))
print('''
Update OK !''')
except Exception as exept:
print('ERROR : {0}'.format(update))
print(exept)
sys.exit()
################
# TRUNCATE TABLE OUT
for i, DICT in enumerate(DICT_TAB):
# continue
# Définition des variables
# i = 1
# if i != 2:
# continue
tab_in = DICT_TAB[i]['table_in']
col_in = DICT_TAB[i]['columns_in']
ind_in = DICT_TAB[i]['index_tab']
tabs_out = DICT_TAB[i]['table_out']
for tab_out in reversed(tabs_out):
# continue
sql = "TRUNCATE TABLE {0}.{1} CASCADE".format(schema_fon, tab_out['name'])
print(sql)
con_fon.execute(sql)
for dep in list_dep:
schema_in = dep + '_' + schema_cad
print('''
INIT import data FROM {}
'''.format(schema_in))
################
# IMPORT IN TABLE OUT
for i, DICT in enumerate(DICT_TAB):
# Définition des variables
# i = 1
# if i != 1:
# continue
tab_in = DICT_TAB[i]['table_in']
col_in = DICT_TAB[i]['columns_in']
ind_in = DICT_TAB[i]['index_tab']
tabs_out = DICT_TAB[i]['table_out']
# Import data
print('''
INIT import data FROM {0}........... {1} sec'''.format(tab_in, time_exec(start_time) ))
tmp = pd.read_sql_table(
table_name = tab_in,
con = engine_cad,
schema = schema_in,
columns = col_in + [ind_in],
chunksize = chunk,
)
# Mise en forme des données
# start_time = dt.datetime.today()
if chunk:
DF = pd.concat(tmp, ignore_index=True)
else:
DF = tmp.copy()
DF.drop_duplicates(inplace=True)
del tmp
# DF = optimize_data_frame(DF)
DF.set_index(ind_in, inplace=True)
print('END import data ........... %s sec'%( time_exec(start_time) ))
for tab in tabs_out:
tab_out = tab['name']
dictio = tab['dict']
col_df = tab['columns_in']
col_ad = tab['columns_add']
get_geo = tab['geom']
drp_esc = tab['drop_escape']
unique = tab['unique']
join = tab['join']
# if tab_out == 'parcelles_73':
# break
# continue
print('INIT TABLE {0} ........... {1} sec'.format(tab_out, time_exec(start_time) ))
df = DF[DF.columns.intersection(col_df)].copy()
# df = optimize_data_frame(df)
# del DF; gc.collect()
# Remplacement des espaces dans les chaines de caractères par des 0
df = replace_escape_by_0(df)
if drp_esc:
df_obj = df.select_dtypes(['object'])
df[df_obj.columns] = df_obj.apply(lambda x: x.str.strip())
# df.replace([' '], '', regex=True, inplace=True)
if dictio :
df.rename(columns=dictio, inplace=True)
if join :
for j in join:
if j['bdd'] == 'in' :
# sys.exit()
df = join_data(df, j, schema_in)
if df.empty:
print('df EMPTY ====> next table')
# pass
continue
# Ajout des champs additionnels
if col_ad :
print('INIT addition columns ........... %s sec'%( time_exec(start_time) ))
for key in col_ad.keys():
if key in df.columns:
df[key + '_tmp'] = df[key].copy()
col_ad[key] = [x if x != key else key+'_tmp' for x in col_ad[key]]
aggreg = col_ad[key]
if aggreg :
df[key] = ''
for col in aggreg:
df[key] += df[col].fillna('')
# df[key] = df[aggreg].agg(''.join, axis=1)
# break
else:
df[key] = aggreg
print('ADD column {0} : {1} ........... {2} sec'.format(key,aggreg, time_exec(start_time) ))
# JOINTURE
if join :
for j in join:
if j['bdd'] == 'out' :
# break
# sys.exit()
df = join_data(df, j, schema_in)
if df.empty:
print('df EMPTY ====> next table')
# pass
continue
# sys.exit()
if unique:
df.drop_duplicates(unique['cols'], keep=unique['keep'], inplace=True)
# Conservation des champs utiles à l'insertion en bdd
name_col_out = engine_fon.dialect.get_columns(engine_fon, tab_out, schema=schema_fon)
name_col_out = [ sub['name'] for sub in name_col_out ]
if 'geom' in name_col_out and 'geom' not in df.columns:
name_col_out.remove('geom')
df = df[df.columns.intersection(name_col_out)]
####################
# Read geodataframe
# Dans le cas où un champs géometrique est nécessaire.
if get_geo:
get_geom_parcelle(df=df, get_geo=get_geo, schema=schema_in)
# print('INIT import geodata ........... %s sec'%( time_exec(start_time) ))
# # Définition des variables géometriques
# ind_geo = get_geo['index_geom']
# tab_geo = get_geo['table_geom_in']
# # Get geodata from Postgis
# # sql = "SELECT {0}, geom FROM (SELECT {0}, geom, max(creat_date), max(update_dat) FROM {1}.{2})".format(ind_geo,schema_in,tab_geo)
# sql = """select distinct on (t2.{0})
# t2.{0},
# t1.geom,
# t1.supf::integer as dcntpa -- récupération de la contenance cadastrale associée car présence de géometrie non référencées dans la table "parcelles"
# FROM "{1}".{2} t1
# INNER JOIN (select distinct on ({0}) {0}, max(creat_date) creat_date, max(update_dat) update_dat FROM "{1}".{2} GROUP BY ({0})) t2
# USING ({0}, creat_date, update_dat)""".format(ind_geo, schema_in, tab_geo)
# tmp = gpd.read_postgis(
# sql = sql,
# con = engine_cad,
# geom_col = 'geom',
# crs = crs,
# chunksize = chunk,
# )
# if chunk:
# gdf = gpd.GeoDataFrame(pd.concat(tmp, ignore_index=True))
# else:
# gdf = tmp.copy()
# # del tmp; gc.collect()
# # gdf = tmp.copy()
# del tmp; gc.collect()
# gdf.set_index(ind_geo, inplace=True)
# gdf.index.name = ind_in
# print('END import geodata ........... %s sec'%( time_exec(start_time) ))
# print('INIT merge data - geodata ........... %s sec'%( time_exec(start_time) ))
# if not gdf[gdf.dcntpa.isna()].empty:
# gdf.dcntpa.fillna(0, inplace=True)
# gdf['dcntpa'] = gdf['dcntpa'].astype(df.dtypes['dcntpa'].type)
# # gdf = gdf.merge(df, on = [ind_in, 'dcntpa'], how='left')
# tmp = gdf.merge(df, on = [ind_in, 'dcntpa'], how='right')
# tmp = tmp.set_geometry('geom', drop=True, crs=crs)
# tmp.rename(columns={'geometry': 'geom'}, inplace=True)
# if tmp[tmp.geom.isna()].empty:
# lst_ind_df = tmp[tmp.geom.isna()].index.tolist()
# lst_ind_gdf = gdf.loc[gdf.index.isin(lst_ind_df)].index.tolist()
# tmp.loc[tmp.index.isin(lst_ind_gdf), 'geom'] = gdf.loc[gdf.index.isin(lst_ind_gdf), 'geom']
# del [gdf, df]; gc.collect()
# gdf = tmp.copy()
# del tmp; gc.collect()
# export_data(gdf)
# del gdf; gc.collect()
# récupération de la liste des géometries où l'id est non présentent dans la table parcelles
# lst = gdf[gdf.par_id.isna()].index.tolist()
# # Recomposition des infos principales
# par_id = [l.replace('0','',1) for l in lst]
# gdf.loc[gdf.index.isin(lst), 'par_id'] = par_id
# gdf.loc[gdf.index.isin(lst), 'codcom'] = [l[:5] for l in par_id]
# gdf.loc[gdf.index.isin(lst), 'ccopre'] = [l[5:8] for l in par_id]
# gdf.loc[gdf.index.isin(lst), 'ccosec'] = [l[8:10] for l in par_id]
# gdf.loc[gdf.index.isin(lst), 'dnupla'] = [l[10:14] for l in par_id]
# # gdf.loc[gdf.index.isin(lst), 'vl_id'] = [l[:8] for l in par_id]
# # gdf = gdf[gdf.vl_id.str.len() == 10]
else:
export_data(df)
del df
del DF #; gc.collect()
print('END transfert data FROM département {0} ........... {1} sec'.format(dep, time_exec(start_time) ))
print('END SCRIPT')
sys.exit()
print('NOT EXIT')
0_FONCIER/foncier_insert_cadastre_V2.py
Executable
+613
View File
@@ -0,0 +1,613 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Nom : : foncier_insert_table.py
#Description : Insertion des données cadastrales à la base <foncier>.
#Copyright : 2021, CEN38
#Auteur : Colas Geier
#Version : 2.0
from sqlalchemy import create_engine, text
from geoalchemy2 import Geometry # need for pandas
import datetime as dt
import pandas as pd
import sys
# Parametrage geopandas
import geopandas as gpd
import warnings; warnings.filterwarnings('ignore', 'GeoSeries.isna', UserWarning)
check_duplicates = True
debug = True
# Parametres bdd CADASTRE (in)
# Données de sortie du plugin qgis "Cadastre"
# Les schéma ont une nomenclarture numérique du type "dep_nom"
user_cad = 'postgres' # utilisateur de connexion à la bdd
pwd_cad = 'foncier_test1' # mot de passe de la bdd
adr_cad = '172.17.0.2' # adresse ip de la bdd
port_cad = '5432' # port de la bdd
base_cad = 'postgres' # nom de la bdd
schema_cad = '202007' # nom unique qui compose le nom des schémas cibles de l'année à récupérer
list_dep = ['07', '26', '42', '38'] # liste des départements
# Parametres bdd FONCIER (out)
user_fon = 'colas_geier'
pwd_fon = 'adm1n*fOncier'
adr_fon = '91.134.194.221'
# user_fon = 'postgres' # utilisateur de connexion à la bdd
# pwd_fon = 'tutu' # mot de passe de la bdd
# adr_fon = '192.168.60.9' # adresse ip de la bdd
port_fon = '5432' # port de la bdd
base_fon = 'bd_cen' # nom de la bdd
schema_fon = 'cadastre' # nom du schéma cible dans la bdd
sufx_nom_tab = '_38' # sufixe du nom des tables où insérer la données dans le schéma cadastre (ex: _dep)
# Correspondance entre les tables
epsg = '2154' # code EPSG de la projection géographique en entrée.
crs = 'EPSG:%s'%epsg
chunk = 100000
# Il existe parfois des doublons en raison d'orthographes voisinent (entre autres)
# dans les tables de sortie du du plugin qgis "Cadastre"
FIND_DOUBLON = [{
'tab_in': 'proprietaire', # nom de la table où des doublons se trouvent
'on_col': ['ddenom', 'dprnlp', 'dldnss','jdatnss','ccogrm','dsglpm','dnatpr'] } # nom des champs où des corrections sont nécessaires
]
DICT_TAB = [{
'table_in' : 'proprietaire', # Table source qui provient de la sortie du plugin cadastre de qgis
'index_tab': 'proprietaire', # Pkey de la table source
'columns_in': ['ccodep', 'ccocom', 'dnupro', # Liste exaustive des champs à récupérer dans la table source
'dnuper', 'ccoqua', 'ddenom', 'jdatnss', 'dldnss', 'dsglpm', 'dlign3', 'dlign4', 'dlign5', 'dlign6', 'dnatpr', 'gtoper', 'ccogrm'],
'table_out': [{
'name': 'cptprop{}'.format(sufx_nom_tab), # Nom de la table cible
'geom': None, # Existance d'une géometrie à récupérer pour insertion (Non: None, Oui: cf tab_out parcelle_xx ci-dessous)
'drop_escape': False, # Supprime les champs vides à l'intérieure des chaines de carractères
'columns_in': ['ccodep', 'ccocom', 'dnupro'], # Liste des columns à récupérer en entrée.
'columns_add': {'dnupro': ['ccodep', 'ccocom', 'dnupro']}, # Définition des champs composés devant être ajoutés
'unique': {'cols': ['dnupro'], 'keep': 'first'}, # Champs devant être uniques à l'intérieur de la table en sortie, keep: ['first', 'last', False]
'dict': None, # Dictionnaire pour renommer les champs {'ancien_nom1': 'nouveau_nom1', 'ancien_nom2': 'nouveau_nom2', ...}
'join': [{ # Jointure de tables à réaliser.
'bdd': 'in', # ['in', 'out']: jointure des tables à l'import / avant export
'table': 'suf', # Nom de la table à joindre
'select_cols' : ['ccodep', 'ccocom', 'dnupro'], # Nom des champs à récupérer
'on': ['ccodep', 'ccocom', 'dnupro'], # Nom des champs sur lesquelles faire la jointure
'type': 'concat' # Type de jointure à réaliser ['concat', 'merge', 'isin', 'notin']
# 'concat': concatène les 2 tables; 'merge': merge les 2 tables en fonction du paramètre 'on'
# 'isin': Conservation des données présentent dans les 2 tables en fonction du paramètre 'on'
# 'notin': Conservation des données NON présentent dans la table 'join' en fonction du paramètre 'on'
},{
'bdd': 'in', 'table': 'lots', 'on': ['ccodep', 'ccocom', 'dnupro'], 'type': 'concat',
'select_cols' : ['ccodep', 'ccocom', 'dnuprol'],'dict': {'dnuprol': 'dnupro'}},{
'bdd': 'in', 'table': 'parcelle', 'on': ['ccodep', 'ccocom', 'dnupro'], 'type': 'concat',
'select_cols' : ['ccodep', 'ccocom', 'dnupro']},]
},{
'name': 'proprios{}'.format(sufx_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in': ['ccodep', 'dnuper', 'ccoqua', 'ddenom', 'jdatnss', 'dldnss', 'dsglpm', 'dlign3', 'dlign4', 'dlign5', 'dlign6', 'dnatpr', 'gtoper', 'ccogrm'],
'columns_add': {'dnuper': ['ccodep', 'dnuper']},
'unique': {'cols': ['dnuper'], 'keep': 'first'},
'dict': None,
'join': False
},{
'name': 'r_prop_cptprop{}'.format(sufx_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in': ['ccodep', 'dnuper', 'ccocom', 'dnupro', 'dnomlp', 'dprnlp', 'epxnee', 'dnomcp', 'dprncp', 'ccodro', 'ccodem'],
'columns_add': {
'dnuper': ['ccodep', 'dnuper'],
'dnupro': ['ccodep', 'ccocom', 'dnupro']},
'unique': {'cols': ['dnupro', 'dnuper'], 'keep': 'first'},
'dict': None,
'join': False
},]
},{
'table_in' : 'parcelle',
'index_tab': 'parcelle',
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'ccovoi', 'dparpi', 'dcntpa', 'ccocomm', 'ccoprem', 'ccosecm', 'dnuplam', 'dvoilib', 'type_filiation'],
'table_out': [{
'name': 'vl{}'.format(sufx_nom_tab),
'geom': None,
'drop_escape': False,
'columns_in' : ['ccodep', 'ccocom', 'ccovoi', 'dvoilib'],
'columns_add': {
'vl_id': ['ccodep', 'ccocom', 'ccovoi'],
'geom': None},
'unique': {'cols': ['vl_id'], 'keep': 'first'},
'dict': {'dvoilib': 'libelle'},
'join': [{
'bdd': 'in', 'table': 'voie', 'on': ['ccodep', 'ccocom', 'ccovoi'], 'type': 'concat',
'select_cols' : ['ccodep', 'ccocom', 'codvoi', 'libvoi'],
'dict': {'libvoi': 'libelle', 'codvoi': 'ccovoi'},
}]
},{
'name': 'parcelles{}'.format(sufx_nom_tab),
'geom': {
'table_geom_in': 'geo_parcelle',
'index_geom': 'geo_parcelle'
},
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'ccovoi', 'dparpi', 'dcntpa', 'ccocomm', 'ccoprem', 'ccosecm', 'dnuplam', 'type_filiation'],
'columns_add': {
'par_id': ['ccodep', 'ccocom', 'ccopre','ccosec', 'dnupla'],
'codcom': ['ccodep', 'ccocom'],
'vl_id': ['ccodep', 'ccocom', 'ccovoi'],
'typprop_id': None },
'unique': False,
'dict': {'type_filiation': 'type'},
'join': False
},{
'name': 'lots{}'.format(sufx_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dcntpa'],
'columns_add': {
'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla'],
'par_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla'],
'dnulot': None, },
'unique': False,
'dict': {'dcntpa': 'dcntlo'},
'join': [{'bdd': 'out', 'table': 'parcelles{}'.format(sufx_nom_tab), 'on': ['par_id'], 'type': 'isin',
'select_cols' :['par_id'] }]
},]
},{
'table_in' : 'lots',
'index_tab': 'lots',
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnupdl', 'dcntlo', 'dnuprol'],
'table_out': [{
'name': 'lots{}'.format(sufx_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnupdl', 'dcntlo'],
'columns_add': {
'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'],
'par_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla'],},
'unique': {'cols': ['lot_id'], 'keep': 'first'},
'dict': None,
'join': [{'bdd': 'out', 'table': 'parcelles{}'.format(sufx_nom_tab), 'on': ['par_id'], 'type': 'isin',
'select_cols' :['par_id'] }]
},{
'name': 'lots_natcult{}'.format(sufx_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'],
'columns_add': {
'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'],},
'unique': {'cols': ['lot_id'], 'keep': 'first'},
'dict': None,
'join': [{
'bdd': 'in', 'table': 'suf', 'on': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'], 'type': 'merge',
'select_cols' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot','dsgrpf','cnatsp','dclssf','ccosub','dcntsf'],
},{
'bdd': 'out', 'table': 'lots{}'.format(sufx_nom_tab), 'on': ['lot_id'], 'type': 'isin',
'select_cols' :['lot_id'] }]
},{
'name': 'cadastre{}'.format(sufx_nom_tab),
'geom': None,
'drop_escape': True,
'columns_in' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnuprol'],
'columns_add': {
'lot_id': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot'],
'dnupro': ['ccodep', 'ccocom', 'dnuprol'],},
'unique': {'cols': ['lot_id', 'dnupro'], 'keep': 'first'},
'dict': None,
'join': [{
'bdd': 'in', 'table': 'suf', 'on': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnuprol'], 'type': 'concat',
'select_cols' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnulot', 'dnupro'], 'dict': {'dnupro': 'dnuprol'}
},{
'bdd': 'in', 'table': 'parcelle', 'on': ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnuprol'], 'type': 'concat',
'select_cols' : ['ccodep', 'ccocom', 'ccopre', 'ccosec', 'dnupla', 'dnupro'], 'dict': {'dnupro': 'dnuprol'}
},{
'bdd': 'out', 'table': 'lots{}'.format(sufx_nom_tab), 'on': ['lot_id'], 'type': 'isin',
'select_cols' :['lot_id'] },{
'bdd': 'out', 'table': 'cptprop{}'.format(sufx_nom_tab), 'on': ['dnupro'], 'type': 'isin',
'select_cols' :['dnupro'] },]
},]
}]
################################
########## Fonctions ##########
################################
start_time = dt.datetime.today()
def time_exec (init_time):
time = dt.datetime.today() - init_time
return str(time)
def replace_escape_by_0 (df):
# Remplacement des espaces dans les chaines de caractères par des 0
# if 'ccopre' in df.columns:
# df['ccopre'].replace([None, '', ' '], '000', inplace=True)
cols = ['ccopre', 'ccosec', 'dnupla', 'dparpi', 'dnuplam', 'dclssf', 'ccovoi']
for col in cols:
if col in df.columns:
df[col].replace([' '], '0', regex=True, inplace=True)
return df
def join_data (df, join, schema_in):
# Jointure des données avec une autre table
table = join['table']
bdd = join['bdd']
typ = join['type']
on = join['on']
if bdd == 'out':
con = engine_fon
sch = schema_fon
if bdd == 'in':
con = engine_cad
sch = schema_in
select_col = []
if 'select_cols' in join.keys():
select_col.extend(join['select_cols'])
if 'where' in join.keys():
select_col.extend(join['where'].keys())
tmp = pd.read_sql_table(
table_name = table,
con = con,
schema = sch,
columns = select_col
)
tmp = replace_escape_by_0(tmp)
if 'dict' in join.keys():
tmp.rename(columns=join['dict'], inplace=True)
if 'where' in join.keys():
where = join['where']
for key in where.keys():
tmp = tmp[tmp[key] == where[key] ]
if typ in ['isin', 'notin']:
for d in [df, tmp]:
d['on'] = ''
for col in on:
d['on'] += d[col].astype(str)
if typ == 'isin':
df = df[df['on'].isin(tmp['on'])]
if typ == 'notin':
df = df[~df['on'].isin(tmp['on'])]
df.drop(columns='on', inplace=True)
if typ == 'merge':
df = df.merge(tmp, on = on, how='left')
if typ == 'concat':
df = pd.concat([df,tmp], ignore_index=True).drop_duplicates()
return df
def get_geom_parcelle (df,get_geo,schema):
print('INIT import geodata ........... %s sec'%( time_exec(start_time) ))
# Définition des variables géometriques
ind_geo = get_geo['index_geom']
tab_geo = get_geo['table_geom_in']
sql = """select distinct on (t2.{0})
t2.{0},
t1.geom,
t1.supf::integer as dcntpa -- récupération de la contenance cadastrale associée car présence de géometrie non référencées dans la table "parcelles"
FROM "{1}".{2} t1
INNER JOIN (select distinct on ({0}) {0}, max(creat_date) creat_date, max(update_dat) update_dat FROM "{1}".{2} GROUP BY ({0})) t2
USING ({0}, creat_date, update_dat)""".format(ind_geo, schema, tab_geo)
tmp = gpd.read_postgis(
sql = sql,
con = engine_cad,
geom_col = 'geom',
crs = crs,
chunksize = chunk,
)
if chunk:
gdf = gpd.GeoDataFrame(pd.concat(tmp, ignore_index=True))
else:
gdf = tmp.copy()
del tmp
gdf.set_index(ind_geo, inplace=True)
gdf.index.name = ind_in
print('END import geodata ........... %s sec'%( time_exec(start_time) ))
print('INIT merge data - geodata ........... %s sec'%( time_exec(start_time) ))
if not gdf[gdf.dcntpa.isna()].empty:
gdf.dcntpa.fillna(0, inplace=True)
gdf['dcntpa'] = gdf['dcntpa'].astype(df.dtypes['dcntpa'].type)
tmp = gdf.merge(df, on = [ind_in, 'dcntpa'], how='right')
tmp = tmp.set_geometry('geom', drop=True, crs=crs)
tmp.rename(columns={'geometry': 'geom'}, inplace=True)
if tmp[tmp.geom.isna()].empty:
lst_ind_df = tmp[tmp.geom.isna()].index.tolist()
lst_ind_gdf = gdf.loc[gdf.index.isin(lst_ind_df)].index.tolist()
tmp.loc[tmp.index.isin(lst_ind_gdf), 'geom'] = gdf.loc[gdf.index.isin(lst_ind_gdf), 'geom']
del [gdf, df]
gdf = tmp.copy()
del tmp
export_data(gdf)
def export_data(df):
print('INIT export data TO {0}, {1} ........... {2} sec'.format(tab_out, df.shape[0], time_exec(start_time) ))
rang = [e for e in range(0, df.shape[0], chunk*5)]
cd1 = 'jdatnss' in df.columns
if cd1 :
tmp = df.jdatnss.str.split(r'/',expand=True)
if tmp.shape[1] == 3 and tmp[2].str.len().max() == 4 and tmp[1].astype(int).max() <= 12:
df.jdatnss = ['%s-%s-%s'%(i[2],i[1],i[0]) for i in df.jdatnss.str.split(r'/').sort_index()]
for i, j in enumerate(rang):
if j == max(rang) :
jj = df.shape[0]
else:
jj = rang[i+1]
df_imp = df[j:jj].copy()
print('INIT export data TO {0} ..... {1}/{2} ...... {3} sec'.format(tab_out, jj, df.shape[0], time_exec(start_time) ))
if 'geom' in df.columns and not df[~df['geom'].isna()].empty :
df_imp = df_imp.set_geometry('geom', drop=True, crs=crs)
df_imp.rename(columns={'geometry': 'geom'}, inplace=True)
df_imp.to_postgis(
name = tab_out,
con = engine_fon,
schema = schema_fon,
index = False,
if_exists = 'append',
geom_col = 'geom',
chunksize = chunk,
)
else:
df_imp.to_sql(
name = tab_out,
con = engine_fon,
schema = schema_fon,
index = False,
if_exists = 'append',
chunksize = chunk,
method = 'multi',
)
print('END export data TO {0} ........... {1} sec'.format(tab_out, time_exec(start_time) ))
def optimize_data_frame(df):
columns = df.columns
for col in columns:
# dtype = df[col].dtypes
# if dtype == 'int64' or dtype == 'int32':
len_col = len(df[col].unique())
if len_col <= df.shape[0]*0.8:
df[col] = df[col].astype('category')
return df
# Initiation des connexions bdd
engine_cad = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user_cad,pwd_cad,adr_cad,port_cad,base_cad), echo=False)
engine_fon = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user_fon,pwd_fon,adr_fon,port_fon,base_fon), echo=False)
con_cad = engine_cad.connect()
con_fon = engine_fon.connect()
################################
########## Main ##########
################################
if __name__ == "__main__":
################
# CORRECTION DUPLICATES TABLE_IN
################
if check_duplicates:
for DOUBLON in FIND_DOUBLON:
tab = DOUBLON['tab_in']
on_col = DOUBLON['on_col']
for col in on_col:
for dep in list_dep:
schema_in = dep + '_' + schema_cad
sql = '''
-- il existe des doublons en raison d'orthographes voisines :
-- recherche de ces doublons
SELECT DISTINCT '{0}' as insee_dep, dnuper, string_agg(DISTINCT {1},' / ') as orthographes_voisines
FROM "{2}".{3} GROUP BY dnuper HAVING count(DISTINCT {1}) > 1'''.format(dep, col, schema_in, tab)
df = pd.read_sql(
sql = sql,
con = engine_cad,
)
if df.empty:
print('No duplicate value dep {0} table {1} column {2} ====> next request'.format(dep, tab, col))
continue
for i, row in df.iterrows():
dnuper = row.dnuper
choix = row.orthographes_voisines.split(' / ')
choix = [i.strip() for i in choix]
Question = input("""Des orthographes voisines existent pour l'identifiant : {0}
dans la colonne : {1}.
Les valeurs voisines sont : {2}
Ecrire la mise à jour du champs {1} à enregistrer (c cancel) :""".format(dnuper,col, choix))
if Question.lower() == 'c' or Question.lower() == 'cancel':
continue
update = '''UPDATE "{0}".{1}
SET {2} = '{3}'
WHERE {2} like '{4}%'
AND dnuper = '{5}';'''.format(schema_in, tab, col, Question, "%' OR {} like '".format(col).join(map(str,choix)), dnuper)
try:
con_cad.execute(text(update))
print('''
Update OK !''')
except Exception as exept:
print('ERROR : {0}'.format(update))
print(exept)
sys.exit()
################
# TRUNCATE TABLE OUT
################
if not debug:
for i, DICT in enumerate(DICT_TAB):
tab_in = DICT_TAB[i]['table_in']
col_in = DICT_TAB[i]['columns_in']
ind_in = DICT_TAB[i]['index_tab']
tabs_out = DICT_TAB[i]['table_out']
for tab_out in reversed(tabs_out):
sql = "TRUNCATE TABLE {0}.{1} CASCADE".format(schema_fon, tab_out['name'])
print(sql)
con_fon.execute(sql)
################
# IMPORT IN TABLE OUT
################
for dep in list_dep:
schema_in = dep + '_' + schema_cad
print('''
INIT import data FROM {}
'''.format(schema_in))
for i, DICT in enumerate(DICT_TAB):
tab_in = DICT_TAB[i]['table_in']
col_in = DICT_TAB[i]['columns_in']
ind_in = DICT_TAB[i]['index_tab']
tabs_out = DICT_TAB[i]['table_out']
# Import data
print('''
INIT import data FROM {0}........... {1} sec'''.format(tab_in, time_exec(start_time) ))
tmp = pd.read_sql_table(
table_name = tab_in,
con = engine_cad,
schema = schema_in,
columns = col_in + [ind_in],
chunksize = chunk,
)
# Mise en forme des données
if chunk:
DF = pd.concat(tmp, ignore_index=True)
else:
DF = tmp.copy()
DF.drop_duplicates(inplace=True)
del tmp
DF.set_index(ind_in, inplace=True)
print('END import data ........... %s sec'%( time_exec(start_time) ))
for tab in tabs_out:
tab_out = tab['name']
dictio = tab['dict']
col_df = tab['columns_in']
col_ad = tab['columns_add']
get_geo = tab['geom']
drp_esc = tab['drop_escape']
unique = tab['unique']
join = tab['join']
print('INIT TABLE {0} ........... {1} sec'.format(tab_out, time_exec(start_time) ))
df = DF[DF.columns.intersection(col_df)].copy()
# Remplacement des espaces dans les chaines de caractères par des 0
df = replace_escape_by_0(df)
if drp_esc:
df_obj = df.select_dtypes(['object'])
df[df_obj.columns] = df_obj.apply(lambda x: x.str.strip())
if dictio :
df.rename(columns=dictio, inplace=True)
if join :
for j in join:
if j['bdd'] == 'in' :
df = join_data(df, j, schema_in)
if df.empty:
print('df EMPTY ====> next table')
continue
# Ajout des champs additionnels
if col_ad :
print('INIT addition columns ........... %s sec'%( time_exec(start_time) ))
for key in col_ad.keys():
if key in df.columns:
df[key + '_tmp'] = df[key].copy()
col_ad[key] = [x if x != key else key+'_tmp' for x in col_ad[key]]
aggreg = col_ad[key]
if aggreg :
df[key] = ''
for col in aggreg:
df[key] += df[col].fillna('')
else:
df[key] = aggreg
print('ADD column {0} : {1} ........... {2} sec'.format(key,aggreg, time_exec(start_time) ))
if tab_out == 'proprios%s'%sufx_nom_tab:
add = pd.DataFrame(data={'dnuper': ['%sY99999'%dep], 'ddenom': ['PROPRIETAIRES NON RENSEIGNES']})
df = pd.concat([df,add])
# JOINTURE
if join :
for j in join:
if j['bdd'] == 'out' :
df = join_data(df, j, schema_in)
if df.empty:
print('df EMPTY ====> next table')
continue
if unique:
df.drop_duplicates(unique['cols'], keep=unique['keep'], inplace=True)
# Conservation des champs utiles à l'insertion en bdd
name_col_out = engine_fon.dialect.get_columns(engine_fon, tab_out, schema=schema_fon)
name_col_out = [ sub['name'] for sub in name_col_out ]
if 'geom' in name_col_out and 'geom' not in df.columns:
name_col_out.remove('geom')
df = df[df.columns.intersection(name_col_out)]
####################
# Read geodataframe
# Dans le cas où un champs géometrique est nécessaire.
if get_geo:
get_geom_parcelle(df=df, get_geo=get_geo, schema=schema_in)
else:
export_data(df)
# del df
# Si insertion de données dans r_prop_cptprop,
# attribution d'un compte 'PROPRIETAIRES NON RENSEIGNES' au cptprop sans proprio
if tab_out == 'r_prop_cptprop%s'%sufx_nom_tab:
print('''ATTRIBUTION d'un compte 'PROPRIETAIRES NON RENSEIGNES' au cptprop sans proprio''')
df = pd.read_sql_table(
con = engine_fon,
table_name = 'cptprop%s'%sufx_nom_tab,
schema = schema_fon,
columns = ['dnupro'] )
tmp = pd.read_sql_table(
con = engine_fon,
table_name = 'r_prop_cptprop%s'%sufx_nom_tab,
schema = schema_fon,
columns = ['dnupro'] )
df = df[~df.dnupro.isin(tmp.dnupro)]
if df.empty:
print('TOUS les propriétaires sont renseignés ===> NEXT !')
continue
df['dnuper'] = '%sY99999'%dep
export_data(df)
del df
del DF
print('END transfert data FROM département {0} ........... {1} sec'.format(dep, time_exec(start_time) ))
print('END SCRIPT')
0_FONCIER/foncier_insert_cadastre_V3.py
+631
View File
@@ -0,0 +1,631 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from pycen import con_fon
from sqlalchemy.sql import text
from sqlalchemy.engine import URL
from sqlalchemy import create_engine
import pandas as pd
import geopandas as gpd
# user = 'cgeier'
# pwd = 'adm1n*bdCen'
# adr = '91.134.194.221'
# base = 'cadastre'
user = 'postgres'
pwd = 'foncier_test1'
adr = '172.17.0.2'
base = 'postgres'
url = URL.create('postgresql+psycopg2',
username=user,
password=pwd,
host=adr,
database=base,
)
con_cad = create_engine(url)
# from pycen import con_cad
# sql = '''SELECT * FROM "38_202207".suf WHERE parcelle = '3800740000B0705';'''
# df = pd.read_sql_query(sql,con_cad)
# df.drop_duplicates(inplace=True)
# df.annee = '2020'
# df.to_sql('suf',con_cad,"38_202207",if_exists='append',index=False)
# fadd = '/home/colas/Documents/tmp/FONCIER_FEDE/add_parc.gpkg'
# add = gpd.read_file(fadd)
def recup_cols_table(table,con,schema='38_202207'):
lst_cols = con.dialect.get_columns(con,table,schema)
return [x['name'] for x in lst_cols]
def __get_pkey__(engine,table_name,schema):
pk = engine.dialect.get_pk_constraint(engine,table_name=table_name,schema=schema)
return pk
def _where_parcelle(sql0,schema,list_parid):
if list_parid is not None:
chunk = None
if isinstance(list_parid,str):
list_parid = [list_parid]
LIST_ID = str(tuple(list_parid)).replace(',)',')')
sql1 = 'AND' if 'WHERE' in sql0 else 'WHERE'
sql1 += '''
p.parcelle IN {list_id}
;'''.format(sch=schema,list_id=LIST_ID)
else :
chunk = 200000
sql1 = ';'
sql = sql0 + sql1
df = pd.read_sql_query(text(sql),con=con_cad,chunksize=chunk)
# if chunk is not None:
# for d in df:
# print(d.shape[0])
# d.drop_duplicates(inplace=True)
# print(d.drop_duplicates().shape[0])
# else :
# df.drop_duplicates(inplace=True)
return df
def _get_chunk(df1,df2):
# cptp1 = pd.DataFrame()
list_DF1 = []
for d1 in df1:
list_DF1.append(d1)
DF1 = pd.concat(list_DF1)
# cptp2 = pd.DataFrame()
list_DF2 = []
for d2 in df2:
list_DF2.append(d2)
DF2 = pd.concat(list_DF2)
return pd.concat([DF1,DF2]).drop_duplicates()
def __get_parcelles__(sql0,list_parid):
if list_parid is not None:
chunk = None
if isinstance(list_parid,str):
list_parid = [list_parid]
LIST_ID = str(tuple(list_parid)).replace(',)',')')
sql1 = '''
WHERE t1.geo_parcelle IN {list_id}
;'''.format(list_id=LIST_ID)
else :
chunk = None
sql1 = ';'
sql = sql0 + sql1
# print(text(sql))
return gpd.read_postgis(sql=sql,con=con_cad,chunksize=chunk)
def _get_parcelles1(schema='38_202207',list_parid=None):
sql0 = '''SELECT DISTINCT ON (t1.geo_parcelle)
t1.geo_parcelle,
case when t1.geom is null then t2.geom
else t1.geom
end geom,
substring(p.parcelle from 1 for 2)||substring(p.parcelle from 4 for 12) par_id,
substring(p.parcelle from 1 for 2)||substring(p.parcelle from 4 for 3) codcom,
substring(p.parcelle from 1 for 2) ccodep,
substring(p.parcelle from 4 for 3) ccocom,
substring(p.parcelle from 7 for 3) ccopre,
substring(p.parcelle from 10 for 2) ccosec,
substring(p.parcelle from 12 for 4) dnupla,
p.annee annee_pci,
t1.update_dat,
p.dparpi,
p.dcntpa,
p.ccocomm,
p.ccoprem,
p.ccosecm,
p.dnuplam,
p.ccovoi,
p.ccoriv,
p.type_filiation "type",
substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 3)||p.ccovoi vl_id,
(SELECT STRING_AGG(DISTINCT gtoper::text,',') FROM "{sch}".proprietaire WHERE (ccodep,ccocom,dnupro) = (t3.ccodep,t3.ccocom,t3.dnupro)) gtoper,
(SELECT STRING_AGG(DISTINCT ccogrm::text,',') FROM "{sch}".proprietaire WHERE (ccodep,ccocom,dnupro) = (t3.ccodep,t3.ccocom,t3.dnupro)) ccogrm,
(SELECT STRING_AGG(DISTINCT CONCAT(gtoper::text||COALESCE('_'||ccogrm::text,'')),',') FROM "{sch}".proprietaire WHERE (ccodep,ccocom,dnupro) = (t3.ccodep,t3.ccocom,t3.dnupro)) ccogrm,
(SELECT STRING_AGG(DISTINCT TRIM(ddenom)::text,',') FROM "{sch}".proprietaire WHERE (ccodep,ccocom,dnupro) = (t3.ccodep,t3.ccocom,t3.dnupro)) ddenom
FROM "{sch}".{t1} p
LEFT JOIN "{sch}".parcelle_info t2 ON t2.geo_parcelle = p.parcelle
LEFT JOIN "{sch}"."geo_parcelle" t1 ON t1.geo_parcelle = p.parcelle
LEFT JOIN "{sch}".proprietaire t3 USING (ccodep,ccocom,dnupro)
'''.format(
sch=schema,
t1='parcelle')
return __get_parcelles__(sql0,list_parid)
def _get_parcelles2(schema='38_202207',list_parid=None):
sql0 = '''SELECT DISTINCT ON (t1.geo_parcelle)
t1.geo_parcelle,
substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) par_id,
substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 3) codcom,
substring(t1.geo_parcelle from 1 for 2) ccodep,
substring(t1.geo_parcelle from 4 for 3) ccocom,
substring(t1.geo_parcelle from 7 for 3) ccopre,
substring(t1.geo_parcelle from 10 for 2) ccosec,
substring(t1.geo_parcelle from 12 for 4) dnupla,
t1.annee annee_pci,
t1.update_dat,
p.dparpi,
p.dcntpa,
p.ccocomm,
p.ccoprem,
p.ccosecm,
p.dnuplam,
p.ccovoi,
p.ccoriv,
p.type_filiation "type",
t1.geom,
substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 3)||p.ccovoi vl_id,
(SELECT STRING_AGG(DISTINCT gtoper::text,',') FROM "{sch}".proprietaire WHERE comptecommunal = t3.comptecommunal) gtoper,
(SELECT STRING_AGG(DISTINCT ccogrm::text,',') FROM "{sch}".proprietaire WHERE comptecommunal = t3.comptecommunal) ccogrm,
(SELECT STRING_AGG(DISTINCT CONCAT(gtoper::text||COALESCE('_'||ccogrm::text,'')),',') FROM "{sch}".proprietaire WHERE comptecommunal = t3.comptecommunal) ccogrm,
(SELECT STRING_AGG(DISTINCT TRIM(ddenom)::text,',') FROM "{sch}".proprietaire WHERE comptecommunal = t3.comptecommunal) ddenom
FROM "{sch}"."{t1}" t1
LEFT JOIN ("{sch}".parcelle_info t2
LEFT JOIN "{sch}".proprietaire t3 USING (comptecommunal))
USING (geo_parcelle)
LEFT JOIN "{sch}".parcelle p ON t1.geo_parcelle = p.parcelle
'''.format(
sch=schema,
t1='geo_parcelle')
return __get_parcelles__(sql0,list_parid)
def _get_parcelles(schema='38_202207',list_parid=None):
p1 = _get_parcelles1(schema,list_parid)
print('parcelles from parcelle .......... OK')
p2 = _get_parcelles2(schema,list_parid)
print('parcelles from geo_parcelle ...... OK')
return _get_chunk(p1,p2)
def _get_voie2(schema='38_202207',list_parid=None):
sql0 = '''
SELECT
t1.ccodep||t1.ccocom||p.ccovoi vl_id,
t1.libvoi libelle
FROM "{sch}"."{t1}" t1
JOIN "{sch}".parcelle p USING (voie)
'''.format(
sch=schema,
t1='voie')
return _where_parcelle(sql0,schema,list_parid)
def _get_lots_natcult1(schema='38_202207',list_parid=None):
sql0 = '''
SELECT
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||btrim(dnulot) as lot_id, -- Identifiant du lot (c'est bon, quand il n'y a pas de lot, c'est la même façon de noter l'identifiant)
btrim(dsgrpf) as dsgrpf , -- Sous-groupe de nature de culture
btrim(cnatsp) as cnatsp , -- Code nature de culture spéciale
CASE WHEN trim(dclssf) = '' OR dclssf = '00'
THEN NULL::integer
ELSE ltrim(dclssf,'0')::integer
END AS dclssf, -- Classe dans le groupe et la série tarif
btrim(ccosub) as ccosub , -- Lettres indicatives de la suf
dcntsf -- Contenance de la suf
FROM "{sch}"."{t1}"
'''.format(
sch=schema,
t1='suf')
return _where_parcelle(sql0,schema,list_parid)
def _get_lots_natcult2(schema='38_202207',list_parid=None):
sql0 = '''
SELECT
CASE WHEN TRIM(dnulot) = ''
THEN substring(parcelle from 1 for 2)||substring(parcelle from 4 for 12)||'0000000'
ELSE substring(parcelle from 1 for 2)||substring(parcelle from 4 for 12)||TRIM(dnulot)
END lot_id,
parcelle,
substring(parcelle from 1 for 2)||substring(parcelle from 4 for 12) par_id,
dsgrpf,
cnatsp,
dclssf,
ccosub,
dcntsf
FROM "{sch}"."{t1}"
JOIN "{sch}".parcelle p USING(parcelle)
'''.format(
sch=schema,
t1='suf')
return _where_parcelle(sql0,schema,list_parid)
def _get_lots01(schema='38_202207'):
sql0 = '''
SELECT DISTINCT
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot as lot_id, -- Identifiant du lot character varying(21)
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla as par_id, -- Identifiant de la parcelle
dnulot, -- Numéro du lot
max(dcntlo) as dcntlo -- Contenance cadastrale (m²) on prend contenance maxi quand même lot_id mais dcntlo différentes
FROM "{sch}".{t1}
JOIN "{sch}".geo_parcelle ON lots.parcelle = geo_parcelle.geo_parcelle -- on ne garde que les lots sur des parcelles dont on a la géométrie
WHERE
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot IN
( -- on sélectionne uniquement les lots qui supportent une subdivision fiscale pour exclure les lots batis
SELECT ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot FROM "{sch}".suf WHERE btrim(dnulot) <> '')
GROUP BY ccodep, ccocom, ccopre, ccosec, dnupla, dnulot
'''.format(
sch=schema,
t1='lots')
return _where_parcelle(sql0,schema,None)
def _get_lots02(schema='38_202207'):
sql0 = '''
SELECT
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot as lot_id, -- Identifiant du lot character varying(21)
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla as par_id, -- Identifiant de la parcelle
dnulot, -- Numéro du lot
sum(dcntsf) as dcntlo -- Contenance cadastrale (m²) = On additionne les contenances des différentes suf qui composent le lot
FROM "{sch}".suf
JOIN "{sch}".geo_parcelle ON suf.parcelle = geo_parcelle.geo_parcelle -- on ne garde que les sufs sur des parcelles dont on a la géométrie
WHERE
btrim(dnulot) <> '' AND ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot NOT IN (
SELECT DISTINCT ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot FROM "{sch}".lots WHERE btrim(dnulot)<>'')
GROUP BY ccodep, ccocom, ccopre, ccosec, dnupla, dnulot
'''.format(
sch=schema,
t1='lots')
return _where_parcelle(sql0,schema,None)
def _get_lots03(schema='38_202207'):
sql0 = '''
-- on sélectionne les parcelles qui supportent une subdivision fiscale (unique ou multiple) même si divisées en lots car dans ce cas ce sont surement des copropriétés (lots batis/jardins...)
SELECT DISTINCT
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla as lot_id, -- Identifiant du lot fictif(=id parcelle)
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla as par_id, -- Identifiant de la parcelle
null as dnulot, -- Numéro du lot
dcntpa as dcntlo-- Contenance cadastrale (m²)
FROM "{sch}".parcelle
JOIN "{sch}".geo_parcelle ON parcelle.parcelle = geo_parcelle.geo_parcelle -- on ne garde que les les parcelles dont on a la géométrie
WHERE
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla IN
(SELECT ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||trim(dnulot) FROM "{sch}".suf); -- toutes les parcelles dont dnulot est NULL
'''.format(
sch=schema,
t1='lots')
return _where_parcelle(sql0,schema,None)
def _get_lots1(schema='38_202207'):
l1 = _get_lots01(schema)
l2 = _get_lots02(schema)
l3 = _get_lots03(schema)
return pd.concat([*l1,*l2,*l3])
def _get_lots2(schema='38_202207',list_parid=None):
sql0 = '''set work_mem='265MB';
SELECT DISTINCT
CASE WHEN TRIM(t.dnulot) = '' OR TRIM(t.dnulot) IS NULL
THEN substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 12)||TRIM(t.ccosub)
ELSE substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 12)||TRIM(t.dnulot)
END lot_id,
t.parcelle,
substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 12) par_id,
CASE WHEN TRIM(t.dnulot) = '' OR TRIM(t.dnulot) IS NULL
THEN TRIM(t.ccosub)
ELSE TRIM(t.dnulot)
END dnulot,
CASE WHEN t.dcntsf IS NULL
THEN l.dcntlo
ELSE t.dcntsf
END dcntlo,
geo_sub1.creat_date,
CASE WHEN geo_sub1.geom IS NULL
THEN geo_sub2.geom
ELSE geo_sub1.geom
END geom
FROM "{sch}"."{t1}" t
JOIN "{sch}".parcelle p USING (parcelle)
LEFT JOIN "{sch}".geo_parcelle geo_p ON geo_p.geo_parcelle = p.parcelle
LEFT JOIN "{sch}".lots l USING (parcelle)
--LEFT JOIN "{sch}".lotslocaux ll USING (lots)
LEFT JOIN "{sch}".geo_subdfisc_parcelle geo_sub_p ON p.parcelle = geo_sub_p.geo_parcelle
--LEFT JOIN "{sch}".geo_subdfisc geo_sub USING (geo_subdfisc)
LEFT JOIN "{sch}".geo_subdfisc geo_sub1
ON (geo_sub1.geo_subdfisc = geo_sub_p.geo_subdfisc AND LOWER(geo_sub1.tex) = LOWER(t.ccosub))
LEFT JOIN "{sch}".geo_subdfisc geo_sub2
ON (geo_sub2.geo_subdfisc = geo_sub_p.geo_subdfisc AND TRIM(LOWER(geo_sub2.tex)) = '')
WHERE ST_INTERSECTS(geo_sub1.geom,geo_p.geom) OR ST_INTERSECTS(geo_sub2.geom,geo_p.geom)
'''.format(
sch=schema,
t1='suf')
return _where_parcelle(sql0,schema,list_parid)
def _get_cadastre2(schema='38_202207',list_parid=None):
sql0 = '''
SELECT DISTINCT
CASE WHEN TRIM(t.dnulot) = ''
THEN substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 12)||'0000000'
ELSE substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 12)||TRIM(t.dnulot)
END lot_id,
substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 3)||t.dnupro dnupro
FROM "{sch}"."{t1}" t
JOIN "{sch}".parcelle p
LEFT JOIN "{sch}".lots l USING (parcelle)
USING (parcelle)
'''.format(
sch=schema,
t1='suf')
return _where_parcelle(sql0,schema,list_parid)
def _get_cptprop1(schema='38_202207',list_parid=None):
sql0 = '''
WITH lot as (
SELECT ccodep,ccocom,dnuprol dnupro
FROM "{sch}".lots
)
SELECT DISTINCT
t.ccodep||t.ccocom||t.dnupro dnupro,
t."annee" annee_matrice
FROM "{sch}"."{t1}" t
--JOIN "{sch}".suf s USING (ccodep,ccocom,dnupro)
--JOIN lot l USING (ccodep,ccocom,dnupro)
JOIN "{sch}".parcelle p USING (ccodep,ccocom,dnupro)
'''.format(
sch=schema,
t1='proprietaire')
return _where_parcelle(sql0,schema,list_parid)
def _get_cptprop2(schema='38_202207',list_parid=None):
sql0 = '''
SELECT DISTINCT
substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 3)||t.dnupro dnupro,
t.annee annee_matrice
FROM "{sch}"."{t1}" t
JOIN "{sch}".parcelle p
LEFT JOIN "{sch}".lots l USING (parcelle)
USING (parcelle)
'''.format(
sch=schema,
t1='suf')
return _where_parcelle(sql0,schema,list_parid)
def _get_cptprop(schema='38_202207',list_parid=None):
cptprop1 = _get_cptprop1(schema=schema, list_parid=list_parid)
print('cptprop from proprietaire ... OK')
cptprop2 = _get_cptprop2(schema=schema, list_parid=list_parid)
print('cptprop from suf ............ OK')
return _get_chunk(cptprop1,cptprop2)
def _get_r_prop_cptprop1(schema='38_202207',list_parid=None):
sql0 = '''
SELECT DISTINCT
substring(p.parcelle from 1 for 2)||substring(p.parcelle from 4 for 3)||t.dnupro dnupro,
substring(p.parcelle from 1 for 2)||t.dnuper dnuper,
ccodro,
ccodem
FROM "{sch}"."{t1}" t
JOIN "{sch}".parcelle p USING (ccodep,ccocom, dnupro)
'''.format(
sch=schema,
t1='proprietaire')
return _where_parcelle(sql0,schema,list_parid)
def _get_proprios1(schema='38_202207',list_parid=None):
sql0 = '''
SELECT DISTINCT
t.ccodep||t.dnuper dnuper,
TRIM(ccoqua)::int ccoqua,
TRIM(ddenom) ddenom,
TRIM(dqualp) dqualp,
TRIM(dnomlp) dnomlp,
TRIM(dnomus) dnomus,
TRIM(dprnus) dprnus,
TRIM(dprnlp) dprnlp,
TRIM(epxnee) epxnee,
TRIM(dnomcp) dnomcp,
TRIM(dprncp) dprncp,
TRIM(jdatnss) jdatnss,
TRIM(dldnss) dldnss,
TRIM(dlign3) dlign3,
TRIM(dlign4) dlign4,
TRIM(dlign5) dlign5,
TRIM(dlign6) dlign6,
TRIM(gtoper)::int gtoper,
TRIM(ccogrm)::int ccogrm,
TRIM(dnatpr) dnatpr,
TRIM(dsglpm) dsglpm,
t."annee" annee_matrice
FROM "{sch}"."{t1}" t
JOIN "{sch}".parcelle p USING (ccodep,ccocom, dnupro)
'''.format(
sch=schema,
t1='proprietaire')
return _where_parcelle(sql0,schema,list_parid)
def _to_cadaste_table(df,con,pkey,table,schema):
# Si la clé-primaire de la table est > 1 colonne
if len(pkey) > 1 :
lstid = str(
tuple(
df[pkey]\
.drop_duplicates()\
.itertuples(index=False, name=None)
)
).replace(',)',')')
pk = str(tuple(pkey)).replace("'",'"')
# Si la clé-primaire de la table est 1 colonne
else:
pk = pkey[0]
lstid = str(tuple(df[pk].drop_duplicates()))\
.replace(',)',')')
# Récupération des données déjà en BDD
sql = '''
SELECT * FROM {sch}.{tab}
WHERE {id} IN {lst}
;'''.format(
sch = schema,
tab = table,
id = pk,
lst = lstid
)
indb = pd.read_sql_query(sql,con,index_col=pkey)
# Exclusion des données déjà en base
df.set_index(pkey,inplace=True)
todb = df[
~df.index.isin(indb.index)
].copy()
df.reset_index(drop=False, inplace=True)
todb.reset_index(drop=False, inplace=True)
indb.reset_index(drop=False, inplace=True)
# Envoie du tableau dans la bdd si non vide
if not todb.empty:
# Adaptation des types des champs si nécessaire
dtyp = {}
if 'geom' in todb.columns:
from geoalchemy2 import Geometry
todb = todb.to_wkt()
dtyp = {'geom':Geometry(geometry_type='MULTIPOLYGON',srid=2154)}
cols = todb.columns[todb.columns.isin(indb.columns)]
todb[cols]\
.to_sql(
table,
con,
schema,
if_exists='append',
index=False,
dtype = dtyp
)
print('INSERT %s news data ! OK'%todb.shape[0])
else :
print('NO news data to insert !')
def to_vl(df,con):
table = 'vl'
schema = 'cadastre'
pkey = __get_pkey__(con,table,schema)['constrained_columns']
_to_cadaste_table(df,con,pkey,table,schema)
def to_parcelles(df,con):
table = 'parcelles'
schema = 'cadastre'
pkey = __get_pkey__(con,table,schema)['constrained_columns']
_to_cadaste_table(df,con,pkey,table,schema)
def to_lots(df,con):
table = 'lots'
schema = 'cadastre'
pkey = __get_pkey__(con,table,schema)['constrained_columns']
_to_cadaste_table(df,con,pkey,table,schema)
def to_lots_natcult(df,con):
table = 'lots_natcult'
schema = 'cadastre'
# pkey = __get_pkey__(con,table,schema)['constrained_columns']
pkey = ['lot_id']
_to_cadaste_table(df,con,pkey,table,schema)
def to_cadastre(df,con):
table = 'cadastre'
schema = 'cadastre'
# pkey = __get_pkey__(con,table,schema)['constrained_columns']
pkey = ['lot_id','dnupro']
_to_cadaste_table(df,con,pkey,table,schema)
def to_r_prop_cptprop(df,con):
table = 'r_prop_cptprop'
schema = 'cadastre'
pkey = __get_pkey__(con,table,schema)['constrained_columns']
_to_cadaste_table(df,con,pkey,table,schema)
def to_proprios(df,con):
table = 'proprios'
schema = 'cadastre'
pkey = __get_pkey__(con,table,schema)['constrained_columns']
_to_cadaste_table(df,con,pkey,table,schema)
def to_cptprop(df,con):
table = 'cptprop'
schema = 'cadastre'
pkey = __get_pkey__(con,table,schema)['constrained_columns']
_to_cadaste_table(df,con,pkey,table,schema)
if __name__ == "__main__":
from pycen import update_to_sql
# par = '3805050000E0523'
par = None
sch = '38_202007'
# cptprop1 = _get_cptprop1(schema=sch, list_parid=par)
# cptprop2 = _get_cptprop2(schema=sch, list_parid=par)
cptprop = _get_cptprop(schema=sch, list_parid=par)
proprios = _get_proprios1(schema=sch, list_parid=par)
r_prop_cptprop = _get_r_prop_cptprop1(schema=sch, list_parid=par)
voie = _get_voie2(schema=sch, list_parid=par)
# parcelles = _get_parcelles2(schema=sch, list_parid=par)
parcelles = _get_parcelles(schema=sch, list_parid=par)
lots = _get_lots2(schema=sch, list_parid=par)
lots_natcult = _get_lots_natcult2(schema=sch, list_parid=par)
cadastre = _get_cadastre2(schema=sch, list_parid=par)
to_cptprop(cptprop,con_fon)
to_proprios(proprios,con_fon)
to_r_prop_cptprop(r_prop_cptprop,con_fon)
to_vl(voie,con_fon)
to_parcelles(parcelles,con_fon)
to_lots(lots,con_fon)
to_lots_natcult(lots_natcult,con_fon)
to_cadastre(cadastre,con_fon)
0_FONCIER/foncier_insert_cadastre_V4.py
+855
View File
@@ -0,0 +1,855 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from pycen import con_fon
from sqlalchemy.sql import text
from sqlalchemy.engine import URL
from sqlalchemy import create_engine
import pandas as pd
import geopandas as gpd
user = 'postgres'
pwd = 'foncier_test1'
adr = '172.17.0.2'
base = 'postgres'
url = URL.create('postgresql+psycopg2',
username=user,
password=pwd,
host=adr,
database=base,
)
con_cad = create_engine(url)
def recup_cols_table(table,con,schema='38_202207'):
lst_cols = con.dialect.get_columns(con,table,schema)
return [x['name'] for x in lst_cols]
def __get_pkey__(engine,table_name,schema):
pk = engine.dialect.get_pk_constraint(engine,table_name=table_name,schema=schema)
return pk
def _where_parcelle(sql0,schema,list_parid):
if list_parid is not None:
chunk = None
if isinstance(list_parid,str):
list_parid = [list_parid]
LIST_ID = str(tuple(list_parid)).replace(',)',')')
sql1 = '''
WHERE p.parcelle IN {list_id}
;'''.format(sch=schema,list_id=LIST_ID)
else :
chunk = 200000
sql1 = ';'
sql = sql0 + sql1
df = pd.read_sql_query(text(sql),con=con_cad,chunksize=chunk)
# if chunk is not None:
# for d in df:
# print(d.shape[0])
# d.drop_duplicates(inplace=True)
# print(d.drop_duplicates().shape[0])
# else :
# df.drop_duplicates(inplace=True)
return df
def _get_chunk(df1,df2):
# cptp1 = pd.DataFrame()
list_DF1 = []
for d1 in df1:
list_DF1.append(d1)
DF1 = pd.concat(list_DF1)
# cptp2 = pd.DataFrame()
list_DF2 = []
for d2 in df2:
list_DF2.append(d2)
DF2 = pd.concat(list_DF2)
return pd.concat([DF1,DF2]).drop_duplicates()
def __get_parcelles__(sql0,list_parid):
if list_parid is not None:
chunk = None
if isinstance(list_parid,str):
list_parid = [list_parid]
LIST_ID = str(tuple(list_parid)).replace(',)',')')
sql1 = '''
WHERE t1.geo_parcelle IN {list_id}
;'''.format(list_id=LIST_ID)
else :
chunk = None
sql1 = ';'
sql = sql0 + sql1
# print(text(sql))
return gpd.read_postgis(sql=sql,con=con_cad,chunksize=chunk)
def _get_parcelles1(schema='38_202207',list_parid=None):
sql0 = '''SELECT DISTINCT ON (t1.geo_parcelle)
t1.geo_parcelle,
case when t1.geom is null then t2.geom
else t1.geom
end geom,
substring(p.parcelle from 1 for 2)||substring(p.parcelle from 4 for 12) par_id,
substring(p.parcelle from 1 for 2)||substring(p.parcelle from 4 for 3) codcom,
substring(p.parcelle from 1 for 2) ccodep,
substring(p.parcelle from 4 for 3) ccocom,
substring(p.parcelle from 7 for 3) ccopre,
substring(p.parcelle from 10 for 2) ccosec,
substring(p.parcelle from 12 for 4) dnupla,
p.annee annee_pci,
t1.update_dat,
p.dparpi,
p.dcntpa,
p.ccocomm,
p.ccoprem,
p.ccosecm,
p.dnuplam,
p.ccovoi,
p.ccoriv,
p.type_filiation "type",
substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 3)||p.ccovoi vl_id,
(SELECT STRING_AGG(DISTINCT gtoper::text,',') FROM "{sch}".proprietaire WHERE (ccodep,ccocom,dnupro) = (t3.ccodep,t3.ccocom,t3.dnupro)) gtoper,
(SELECT STRING_AGG(DISTINCT ccogrm::text,',') FROM "{sch}".proprietaire WHERE (ccodep,ccocom,dnupro) = (t3.ccodep,t3.ccocom,t3.dnupro)) ccogrm,
(SELECT STRING_AGG(DISTINCT CONCAT(gtoper::text||COALESCE('_'||ccogrm::text,'')),',') FROM "{sch}".proprietaire WHERE (ccodep,ccocom,dnupro) = (t3.ccodep,t3.ccocom,t3.dnupro)) ccogrm,
(SELECT STRING_AGG(DISTINCT TRIM(ddenom)::text,',') FROM "{sch}".proprietaire WHERE (ccodep,ccocom,dnupro) = (t3.ccodep,t3.ccocom,t3.dnupro)) ddenom
FROM "{sch}".{t1} p
LEFT JOIN "{sch}".parcelle_info t2 ON t2.geo_parcelle = p.parcelle
LEFT JOIN "{sch}"."geo_parcelle" t1 ON t1.geo_parcelle = p.parcelle
LEFT JOIN "{sch}".proprietaire t3 USING (ccodep,ccocom,dnupro)
'''.format(
sch=schema,
t1='parcelle')
return __get_parcelles__(sql0,list_parid)
def drop_tables(con):
sql = '''
TRUNCATE TABLE cadastre.parcelles RESTART IDENTITY CASCADE;
TRUNCATE TABLE cadastre.cadastre RESTART IDENTITY CASCADE;
TRUNCATE TABLE cadastre.cptprop RESTART IDENTITY CASCADE;
TRUNCATE TABLE cadastre.r_prop_cptprop RESTART IDENTITY CASCADE;
TRUNCATE TABLE cadastre.lots RESTART IDENTITY CASCADE;
TRUNCATE TABLE cadastre.lots_natcult RESTART IDENTITY CASCADE;
TRUNCATE TABLE cadastre.proprios RESTART IDENTITY CASCADE;
TRUNCATE TABLE cadastre.vl RESTART IDENTITY CASCADE;
ALTER SEQUENCE
cadastre.cadastre_cad_id_seq RESTART WITH 1;
ALTER SEQUENCE
cadastre.lots_natcult_lotnatcult_id_seq RESTART WITH 1;
'''
with con.begin() as cnx:
cnx.execute(sql)
def where_parid(parid):
if not parid: return ""
return "AND parcelle.parcelle {} {}".format(
"IN" if isinstance(parid, (list,pd.Series)) and len(parid) > 1 else "=",
tuple(parid) if isinstance(parid, (list,pd.Series)) and len(parid) > 1
else "'%s'"%parid[0] if isinstance(parid, (list,pd.Series)) and len(parid) == 1
else "'%s'"%parid
)
# CENRA 2020
def _insert_voie1(schema='38_202207',list_parid=None):
'''Insertion des voies et lieux-dits dans cadastre.vl à partir de voie.
On ne garde que les voies qui sont effectivement affectées à des parcelles pour ne pas se trainer les voies annulées (annul = 'O')
'''
sql0 = '''set work_mem='512MB';
INSERT into cadastre.vl
SELECT
v.ccodep||v.ccocom||v.codvoi AS vl_id, --character varying(10) NOT NULL, -->>> c'est pas le bon compte de caractères, mais je n'ai pas de doublons en auvergne
v.libvoi AS libelle,--character varying(50),
null AS geom -- geom geometry(MultiPolygon,2154),
FROM "{sch}".voie v
LEFT JOIN "{sch}".parcelle ON v.ccodep||v.ccocom||v.codvoi = parcelle.ccodep||parcelle.ccocom||parcelle.ccovoi
WHERE v.ccodep||v.ccocom||v.codvoi IN (
SELECT DISTINCT ccodep||ccocom||ccovoi FROM "{sch}".parcelle )
'''.format(
sch=schema)
if list_parid:
sql0 += where_parid(list_parid)
with con_fon.begin() as cnx:
cnx.execute(sql0)
def _insert_voie2(schema='38_202207',list_parid=None):
'''Insertions des vois manquantes à partir de leur ccoriv --> 12
'''
sql0 = '''set work_mem='512MB';
INSERT into cadastre.vl
SELECT DISTINCT
parcelle.ccodep||parcelle.ccocom||parcelle.ccovoi AS vl_id,
libvoi
FROM "{sch}".parcelle
JOIN "{sch}".voie ON voie.ccoriv = parcelle.ccoriv AND voie.ccocom = parcelle.ccocom
WHERE parcelle.ccodep||parcelle.ccocom||parcelle.ccovoi NOT IN (
SELECT ccodep||ccocom||codvoi AS vl_id FROM "{sch}".voie)
'''.format(
sch=schema)
if list_parid:
sql0 += where_parid(list_parid)
with con_fon.begin() as cnx:
cnx.execute(sql0)
def _insert_voie3(schema='38_202207',list_parid=None):
'''Ajout manuel des dernières voies référencées dans parcelle mais introuvables dans voie (ni par codvoi ni par ccoriv) --> 0
'''
sql0 = '''set work_mem='512MB';
INSERT into cadastre.vl
SELECT DISTINCT ccodep||ccocom||ccovoi AS vl_id, 'NON TROUVEE' AS libelle
FROM "{sch}".parcelle
WHERE ccodep||ccocom||ccovoi NOT IN (SELECT vl_id from cadastre.vl)
'''.format(
sch=schema)
if list_parid:
sql0 += where_parid(list_parid)
with con_fon.begin() as cnx:
cnx.execute(sql0)
def insert_voie(schema='38_202207',list_parid=None):
_insert_voie1(schema,list_parid)
_insert_voie2(schema,list_parid)
_insert_voie3(schema,list_parid)
print('INSERT voie OK')
# def _manage_parc_without_geom(schema='38_202207'):
# sql = '''CREATE TEMP TABLE parcelles_sans_geom AS (
# SELECT parcelle FROM "{sch}".parcelle a LEFT JOIN "{sch}".geo_parcelle b ON b.geo_parcelle = a.parcelle WHERE b.geo_parcelle is null
# '''.format(
# sch=schema)
# with con_fon.begin() as cnx:
# cnx.execute(sql)
# def add_old_geom(schema='38_202207'):
# sql = '''
# '''
def _insert_parcelle1(schema='38_202207',list_parid=None):
'''01/ parcelles'''
sql = '''set work_mem='512MB';
INSERT INTO cadastre.parcelles
(
SELECT DISTINCT ON (ccodep || ccocom || replace(ccopre, ' ', '0') || replace(ccosec, ' ', '0') || dnupla)
ccodep || ccocom || replace(ccopre, ' ', '0') || replace(ccosec, ' ', '0') || dnupla AS par_id,
-- Identifiant de la parcelle
ccodep || ccocom AS codcom,
-- Code insee de la commune
--CASE WHEN btrim(ccovoi) <> '' THEN ccodep || ccocom || ccovoi ELSE NULL END AS vl_id,
ccodep || ccocom || ccovoi AS vl_id,
--character varying(10) NOT NULL, -->>> on ne met rien lorsque que ccovoi non renseigné dans la table parcelle
btrim(ccopre) as ccopre,
-- Préfixe de section ou quartier servi
btrim(ccosec) as ccosec,
-- Code de la section
CASE WHEN btrim(dnupla) = ''
THEN NULL :: INTEGER
ELSE ltrim(btrim(dnupla), '0') :: INTEGER END AS dnupla,
-- Numéro de la parcelle
CASE WHEN btrim(dparpi) = ''
THEN NULL :: INTEGER
ELSE ltrim(btrim(dparpi), '0') :: INTEGER END AS dparpi,
-- Ancien numéro de la parcelle, parcelle primitive
dcntpa :: INTEGER,
-- Contenance cadastrale de la parcelle
NULL AS typprop_id,
-- Type de propriété de la parcelle >>>>>>>>>>>>>>>>>>>>>>>>>>>>>> à remplir après (Cf. tâches de mise à jour du cadastre)
geom,
-- Géométrie de la parcelle
ccocomm,
-- Commune Parcelle mère
ccoprem,
-- Préfixe Parcelle mère
btrim(ccosecm) as ccosecm,
-- Section Parcelle mère
CASE WHEN dnuplam = ''
THEN NULL :: INTEGER
ELSE ltrim(dnuplam, '0') :: INTEGER
END AS dnuplam,
-- Numéro Parcelle mère
type_filiation AS "type", -- Type de filiation
jdatat
FROM "{sch}".parcelle
JOIN "{sch}".geo_parcelle ON geo_parcelle.geo_parcelle = parcelle.parcelle
'''.format(
sch=schema)
if list_parid:
sql += where_parid(list_parid).replace('AND','WHERE')
sql += ');'
with con_fon.begin() as cnx:
cnx.execute(sql)
# A REPRENDRE
def _insert_parcelle2(schema='38_202207',list_parid=None):
'''2/ Géométries manquantes
Parcelles MAJIC qui n'ont plus de geom dans EDIGEO parce qu'elles ont été fusionnées, divisées ou transférées
Il y a la geom des nouvelles parcelles dans EDIGEO mais rien dans MAJIC
On récupère la geom sur un EDIGEO antérieur'''
sql = '''set work_mem='512MB';
INSERT INTO cadastre.parcelles
( WITH t1 as (
SELECT DISTINCT ON (a.ccodep || a.ccocom || replace(a.ccopre, ' ', '0') || replace(a.ccosec, ' ', '0') || a.dnupla)
parcelle,
a.ccodep || a.ccocom || replace(a.ccopre, ' ', '0') || replace(a.ccosec, ' ', '0') || a.dnupla AS par_id,
-- Identifiant de la parcelle
a.ccodep || a.ccocom AS codcom,
-- Code insee de la commune
--CASE WHEN btrim(ccovoi) <> '' THEN ccodep || ccocom || ccovoi ELSE NULL END AS vl_id,
a.ccodep || a.ccocom || a.ccovoi AS vl_id,
--character varying(10) NOT NULL, -->>> on ne met rien lorsque que ccovoi non renseigné dans la table parcelle
replace(a.ccopre, ' ', '0') AS ccopre,
-- Préfixe de section ou quartier servi
-- btrim(a.ccosec) AS ccosec,
replace(a.ccosec, ' ', '0') AS ccosec,
-- Code de la section
CASE WHEN btrim(a.dnupla) = ''
THEN NULL :: INTEGER
ELSE ltrim(btrim(a.dnupla), '0') :: INTEGER
END AS dnupla,
-- Numéro de la parcelle
CASE WHEN btrim(a.dparpi) = ''
THEN NULL :: INTEGER
ELSE ltrim(btrim(a.dparpi), '0') :: INTEGER
END AS dparpi,
-- Ancien numéro de la parcelle, parcelle primitive
a.dcntpa :: INTEGER,
-- Contenance cadastrale de la parcelle
NULL AS typprop_id,
-- Type de propriété de la parcelle >>>>>>>>>>>>>>>>>>>>>>>>>>>>>> à remplir après (Cf. tâches de mise à jour du cadastre)
-- parcelles_cen.geom,
-- Géométrie de la parcelle
a.ccocomm,
-- Commune Parcelle mère
a.ccoprem,
-- Préfixe Parcelle mère
btrim(a.ccosecm) AS ccosecm,
-- Section Parcelle mère
CASE WHEN a.dnuplam = ''
THEN NULL :: INTEGER
ELSE ltrim(a.dnuplam, '0') :: INTEGER END AS dnuplam,
-- Numéro Parcelle mère
a.type_filiation AS "type", -- Type de filiation
a.jdatat
FROM "{sch}".parcelle a )
SELECT
t1.par_id,
t1.codcom,
t1.vl_id,
t1.ccopre,
t1.ccosec,
t1.dnupla,
t1.dparpi,
t1.dcntpa,
t1.typprop_id,
CASE WHEN b.geom IS NULL THEN parcelles_cen.geom
ELSE b.geom END geom,
t1.ccocomm,
t1.ccoprem,
t1.ccosecm,
t1.dnuplam,
t1.type,
t1.jdatat
FROM t1
LEFT JOIN "{sch}".parcelle_info b ON t1.parcelle = b.geo_parcelle -- les parcelles sans geom
LEFT JOIN "{sch}".parcelle c ON t1.parcelle = c.parcelle -- les parcelles sans geom
LEFT JOIN cadastre.parcelles_cen ON t1.par_id = parcelles_cen.par_id
WHERE t1.par_id NOT IN (SELECT par_id FROM cadastre.parcelles)
)
'''.format(
sch=schema)
if list_parid:
sql += where_parid(list_parid)
with con_fon.begin() as cnx:
cnx.execute(sql)
def insert_parcelle(schema='38_202207',list_parid=None):
_insert_parcelle1(schema,list_parid)
_insert_parcelle2(schema,list_parid)
print('INSERT parcelle OK')
def checkortho_proprio_sym(con,colonne,schema='38_202207'):
sql = '''
SELECT DISTINCT '{dep}' AS insee_dep, dnuper, string_agg(DISTINCT TRIM({column}),' / ') AS orthographes_voisines
FROM "{sch}".proprietaire
GROUP BY dnuper HAVING count(DISTINCT TRIM({column})) > 1
'''.format(dep = schema[:2],
sch=schema,
column=colonne)
return pd.read_sql(sql,con=con)
def update_synonyme_proprio(con,data,colonne,schema='38_202207'):
data['split_ortho'] = (data.orthographes_voisines
.str.replace("'","''")
.str.split(' / '))
for i,row in data.iterrows():
sql = '''UPDATE "{sch}".proprietaire SET {column} = '{dd1}' WHERE btrim({column}) {sym} {dd2} AND dnuper = '{dnup}';'''.format(
sch=schema,
dnup=row.dnuper,
column=colonne,
dd1=row.split_ortho[0],
sym='IN' if len(row.split_ortho[1:]) > 1 else '=',
dd2=tuple(row.split_ortho[1:]) if len(row.split_ortho[1:]) > 1 else "'%s'"% row.split_ortho[1])
with con.begin() as cnx:
cnx.execute(sql)
def check_proprio(schema='38_202207',con=con_fon):
is_synonyme = checkortho_proprio_sym(con,'ddenom',schema)
if not is_synonyme.empty:
update_synonyme_proprio(con,is_synonyme,'ddenom',schema)
is_diff_lab = checkortho_proprio_sym(con,'dldnss',schema)
if not is_diff_lab.empty:
update_synonyme_proprio(con,is_diff_lab,'dldnss',schema)
is_incomplete = checkortho_proprio_sym(con,'jdatnss',schema)
if not is_incomplete.empty:
update_synonyme_proprio(con,is_incomplete,'jdatnss',schema)
is_diff_ccogrm = checkortho_proprio_sym(con,'ccogrm',schema)
if not is_diff_ccogrm.empty:
update_synonyme_proprio(con,is_diff_ccogrm,'ccogrm',schema)
is_diff_dsglpm = checkortho_proprio_sym(con,'dsglpm',schema)
if not is_diff_dsglpm.empty:
update_synonyme_proprio(con,is_diff_dsglpm,'dsglpm',schema)
is_diff_dnatpr = checkortho_proprio_sym(con,'dnatpr',schema)
if not is_diff_dnatpr.empty:
update_synonyme_proprio(con,is_diff_dnatpr,'dnatpr',schema)
is_diff_dprnus = checkortho_proprio_sym(con,'dprnus',schema)
if not is_diff_dprnus.empty:
update_synonyme_proprio(con,is_diff_dprnus,'dprnus',schema)
is_diff_dprnlp = checkortho_proprio_sym(con,'dprnlp',schema)
if not is_diff_dprnlp.empty:
update_synonyme_proprio(con,is_diff_dprnlp,'dprnlp',schema)
is_diff_dnomus = checkortho_proprio_sym(con,'dnomus',schema)
if not is_diff_dnomus.empty:
update_synonyme_proprio(con,is_diff_dnomus,'dnomus',schema)
def _insert_proprio(schema='38_202207',list_parid=None):
sql = '''set work_mem='512MB';
INSERT INTO cadastre.proprios
SELECT DISTINCT
ccodep||dnuper dnuper,
ccoqua::integer,
btrim(ddenom) AS ddenom,
jdatnss,
btrim(dldnss) AS dldnss,
btrim(dsglpm) AS dsglpm,
btrim(dlign3) AS dlign3,
btrim(dlign4) AS dlign4,
btrim(dlign5) AS dlign5,
btrim(dlign6) AS dlign6,
btrim(dnatpr) AS dnatpr,
gtoper::integer,
ccogrm::integer,
btrim(dqualp) AS dnatpr,
btrim(dnomlp) AS dnomlp,
btrim(dprnlp) AS dprnlp,
btrim(dnomus) AS dnomus,
btrim(dprnus) AS dprnus,
btrim(dforme) AS dforme
FROM "{sch}".proprietaire
'''.format(
sch=schema)
with con_fon.begin() as cnx:
cnx.execute(sql)
def insert_no_proprio(schema='38_202207'):
sql = '''
INSERT INTO cadastre.proprios (dnuper, ddenom)
VALUES ('{dep}Y99999', ' PROPRIETAIRES NON RENSEIGNES');
'''.format(
dep=schema[:2])
with con_fon.begin() as cnx:
cnx.execute(sql)
def insert_proprio(schema='38_202207'):
check_proprio(schema,con_fon)
_insert_proprio(schema)
insert_no_proprio(schema)
print('INSERT proprio OK')
def _insert_cptprop1(schema='38_202207'):
'''
Insertion des comptes de propriété depuis la table proprietaire
'''
sql = '''set work_mem='512MB';
INSERT INTO cadastre.cptprop
SELECT DISTINCT ccodep || ccocom || dnupro
FROM "{sch}".proprietaire WHERE btrim(dnupro) <> ''
'''.format(
sch=schema)
with con_fon.begin() as cnx:
cnx.execute(sql)
def _insert_cptprop2(schema='38_202207'):
'''
Insertion des comptes de propriété utilisés dans
lots/parcelles/suf mais non référencés dans proprietaire
'''
sql = '''set work_mem='512MB';
INSERT INTO cadastre.cptprop
SELECT DISTINCT ccodep || ccocom || dnuprol
FROM "{sch}".lots WHERE comptecommunal NOT IN (SELECT DISTINCT comptecommunal FROM "{sch}".proprietaire)
UNION
SELECT DISTINCT ccodep || ccocom || dnupro
FROM "{sch}".parcelle WHERE comptecommunal NOT IN (SELECT DISTINCT comptecommunal FROM "{sch}".proprietaire)
UNION
SELECT DISTINCT ccodep || ccocom || dnupro
FROM "{sch}".suf WHERE comptecommunal NOT IN (SELECT DISTINCT comptecommunal FROM "{sch}".proprietaire)
'''.format(
sch=schema)
with con_fon.begin() as cnx:
cnx.execute(sql)
def insert_cptprop(schema='38_202207'):
_insert_cptprop1(schema)
_insert_cptprop2(schema)
print('INSERT cptprop OK')
def _insert_r_prop_cptprop0(schema='38_202207'):
sql = '''set work_mem='512MB';
INSERT INTO cadastre.r_prop_cptprop
SELECT DISTINCT
ccodep||dnuper,
ccodep || ccocom || dnupro,
btrim(dnomlp),
btrim(dprnlp),
CASE WHEN btrim(epxnee) = ''
THEN NULL::text
ELSE btrim(epxnee)
END AS epxnee ,
btrim(dnomcp),
btrim(dprncp),
btrim(ccodro),
btrim(ccodem)
FROM "{sch}".proprietaire WHERE btrim(dnupro) <> ''
'''.format(
sch=schema)
with con_fon.begin() as cnx:
cnx.execute(sql)
def _insert_r_prop_cptprop1(schema='38_202207'):
sql = '''set work_mem='512MB';
INSERT INTO cadastre.r_prop_cptprop
SELECT DISTINCT '{dep}Y99999', ccodep || ccocom || dnuprol
FROM "{sch}".lots WHERE comptecommunal NOT IN (SELECT DISTINCT comptecommunal FROM "{sch}".proprietaire)
UNION
SELECT DISTINCT '{dep}Y99999', ccodep || ccocom || dnupro
FROM "{sch}".parcelle WHERE comptecommunal NOT IN (SELECT DISTINCT comptecommunal FROM "{sch}".proprietaire)
UNION
SELECT DISTINCT '{dep}Y99999', ccodep || ccocom || dnupro
FROM "{sch}".suf WHERE comptecommunal NOT IN (SELECT DISTINCT comptecommunal FROM "{sch}".proprietaire)
'''.format(
sch=schema,
dep=schema[:2]
)
with con_fon.begin() as cnx:
cnx.execute(sql)
def insert_r_prop_cptprop(schema='38_202207'):
_insert_r_prop_cptprop0(schema)
_insert_r_prop_cptprop1(schema)
print('INSERT r_prop_cptprop OK')
def _insert_lot1(schema='38_202207',list_parid=None):
sql = '''set work_mem='512MB';
INSERT INTO cadastre.lots (lot_id, par_id, dnulot, dcntlo)
--les parcelles divisées en lots de la table lots
(WITH parcelle AS (
SELECT DISTINCT
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot AS id_lot, -- Identifiant du lot character varying(21)
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla AS par_id, -- Identifiant de la parcelle
dnulot, -- Numéro du lot
max(dcntlo) AS dcntlo -- Contenance cadastrale (m²) on prend contenance maxi quand même id_lot mais dcntlo différentes
FROM "{sch}".lots
JOIN "{sch}".geo_parcelle ON lots.parcelle = geo_parcelle.geo_parcelle -- on ne garde que les lots sur des parcelles dont on a la géométrie
WHERE ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot IN
( -- on sélectionne uniquement les lots qui supportent une subdivision fiscale pour exclure les lots batis
SELECT ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot FROM "{sch}".suf WHERE btrim(dnulot) <> '')
GROUP BY ccodep, ccocom, ccopre, ccosec, dnupla, dnulot
UNION -- les lots de la table suf qui ne sont pas dans la table lots
SELECT DISTINCT
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot AS id_lot, -- Identifiant du lot character varying(21)
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla AS par_id, -- Identifiant de la parcelle
dnulot, -- Numéro du lot
sum(dcntsf) AS dcntlo -- Contenance cadastrale (m²) = On additionne les contenances des différentes suf qui composent le lot
FROM "{sch}".suf
JOIN "{sch}".geo_parcelle ON suf.parcelle = geo_parcelle.geo_parcelle -- on ne garde que les sufs sur des parcelles dont on a la géométrie
WHERE btrim(dnulot) <> '' AND ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot NOT IN (
SELECT DISTINCT ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot FROM "{sch}".lots WHERE btrim(dnulot)<>'')
GROUP BY ccodep, ccocom, ccopre, ccosec, dnupla, dnulot
UNION -- les parcelles à lot unique
-- on sélectionne les parcelles qui supportent une subdivision fiscale (unique ou multiple) même si divisées en lots car dans ce cas ce sont surement des copropriétés (lots batis/jardins...)
SELECT DISTINCT
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla AS id_lot, -- Identifiant du lot fictif(=id parcelle)
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla AS par_id, -- Identifiant de la parcelle
null AS dnulot, -- Numéro du lot
dcntpa AS dcntlo-- Contenance cadastrale (m²)
FROM "{sch}".parcelle
JOIN "{sch}".geo_parcelle ON parcelle.parcelle = geo_parcelle.geo_parcelle -- on ne garde que les parcelles dont on a la géométrie
WHERE ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla IN
(SELECT ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||trim(dnulot) FROM "{sch}".suf) -- toutes les parcelles dont dnulot est NULL
)
SELECT id_lot, par_id, dnulot, dcntlo
FROM parcelle
'''.format(
sch=schema)
if list_parid:
list_parid = [x[:2] + x[3:] for x in list_parid]
sql += (where_parid(list_parid)
.replace('AND', 'WHERE')
.replace('.parcelle','.par_id')
)
sql += ');'
with con_fon.begin() as cnx:
cnx.execute(sql)
def _insert_lot2(schema='38_202207'):
sql = '''set work_mem='512MB';
insert into cadastre.lots (lot_id, par_id, dnulot, dcntlo,geom)
with t1 as (
SELECT DISTINCT
-- CASE WHEN TRIM(t.dnulot) = '' OR TRIM(t.dnulot) IS NULL
-- THEN substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 12)||TRIM(t.ccosub)
-- ELSE substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 12)||TRIM(t.dnulot)
-- END lot_id,
--substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 12)||TRIM(t.dnulot) lot_id,
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||TRIM(dnupla)||TRIM(dnulot) AS lot_id,
parcelle,
t.ccosub,
t.dcntsf,
--substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 12) par_id,
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||TRIM(dnupla) AS par_id,
CASE WHEN TRIM(t.dnulot) = '' OR TRIM(t.dnulot) IS NULL
THEN TRIM(t.ccosub)
ELSE TRIM(t.dnulot)
END dnulot
FROM "{sch}"."suf" t
)
select distinct on (t1.lot_id,t1.par_id)
t1.lot_id,
t1.par_id,
t1.dnulot,
CASE WHEN t1.dcntsf IS NULL
THEN l.dcntlo
ELSE t1.dcntsf
END dcntlo,
CASE WHEN geo_sub1.geom IS NULL
THEN geo_sub2.geom
ELSE geo_sub1.geom
END geom
from t1
JOIN "{sch}".parcelle p USING (parcelle)
LEFT JOIN "{sch}".geo_parcelle geo_p ON geo_p.geo_parcelle = p.parcelle
LEFT JOIN "{sch}".lots l USING (parcelle)
--LEFT JOIN "{sch}".lotslocaux ll USING (lots)
LEFT JOIN "{sch}".geo_subdfisc_parcelle geo_sub_p ON p.parcelle = geo_sub_p.geo_parcelle
--LEFT JOIN "{sch}".geo_subdfisc geo_sub USING (geo_subdfisc)
LEFT JOIN "{sch}".geo_subdfisc geo_sub1
ON (geo_sub1.geo_subdfisc = geo_sub_p.geo_subdfisc AND LOWER(geo_sub1.tex) = LOWER(t1.ccosub))
LEFT JOIN "{sch}".geo_subdfisc geo_sub2
ON (geo_sub2.geo_subdfisc = geo_sub_p.geo_subdfisc AND TRIM(LOWER(geo_sub2.tex)) = '')
WHERE (t1.par_id,t1.lot_id) not IN ( select par_id,lot_id from cadastre.lots )
OR t1.par_id not IN ( select par_id from cadastre.lots )
order by 2,1,5
;
'''.format(
sch=schema)
with con_fon.begin() as cnx:
cnx.execute(sql)
def insert_lot(schema='38_202207'):
_insert_lot1(schema)
_insert_lot2(schema)
print('INSERT lot OK')
def insert_cadastre(schema='38_202207',list_parid=None):
sql = '''
SET work_mem='512MB';
INSERT into cadastre.cadastre (lot_id, dnupro)
( WITH t1 as (
--les pdl
SELECT DISTINCT
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot AS lot_id, -- Identifiant du lot character varying(21)
ccodep || ccocom || dnuprol AS dnupro
FROM "{sch}".lots
JOIN "{sch}".geo_parcelle ON lots.parcelle = geo_parcelle.geo_parcelle -- on ne garde que les lots sur des parcelles dont on a la géométrie
WHERE
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||trim(dnulot) IN ( -- on sélectionne uniquement les lots qui supportent une subdivision fiscale
SELECT ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||trim(dnulot)
FROM "{sch}".suf)
UNION
-- les lots de la table suf qui ne sont pas dans la table lots
SELECT DISTINCT
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||dnulot AS id_lot, -- Identifiant du lot character varying(21)
ccodep || ccocom || dnupro AS dnupro
FROM "{sch}".suf
JOIN "{sch}".geo_parcelle ON suf.parcelle = geo_parcelle.geo_parcelle -- on ne garde que les sufs sur des parcelles dont on a la géométrie
WHERE
trim(dnulot) <> '' AND ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||btrim(dnulot)
NOT IN (
SELECT DISTINCT ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||btrim(dnulot) FROM "{sch}".lots)
UNION
---les parcelles sans lot
SELECT DISTINCT
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla AS id_lot, -- Identifiant du lot fictif(=id parcelle)
ccodep || ccocom || dnupro AS dnupro
FROM "{sch}".parcelle
JOIN "{sch}".geo_parcelle ON parcelle.parcelle = geo_parcelle.geo_parcelle -- on ne garde que les parcelles dont on a la géométrie
WHERE
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla IN
(SELECT ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||btrim(dnulot) FROM "{sch}".suf) -- toutes les parcelles dont dnulot est NULL
)
SELECT lot_id, dnupro FROM t1
'''.format(
sch=schema)
if list_parid:
sql += 'JOIN cadastre.lots l USING (lot_id) '
list_parid = [x[:2] + x[3:] for x in list_parid]
sql += (where_parid(list_parid)
.replace('AND', 'WHERE')
.replace('parcelle.parcelle','l.par_id')
)
sql += ');'
with con_fon.begin() as cnx:
cnx.execute(sql)
print('INSERT cadastre OK')
def cadastre_missing(con=con_fon):
sql = '''set work_mem='512MB';
INSERT into cadastre.cadastre (lot_id, dnupro)
SELECT DISTINCT
lot_id, -- Identifiant du lot character varying(21)
dnupro
FROM cadastre.cadastre_cen
WHERE lot_id NOT IN (SELECT lot_id FROM cadastre.cadastre)
AND lot_id IN (SELECT lot_id FROM cadastre.lots)
AND dnupro IN (SELECT dnupro FROM cadastre.cptprop)
;
'''
with con.begin() as cnx:
cnx.execute(sql)
def insert_lotnatcult(schema='38_202207',list_parid=None):
sql = '''set work_mem='512MB';
INSERT INTO cadastre.lots_natcult (lot_id, dsgrpf, cnatsp, dclssf, ccosub, dcntsf)
SELECT DISTINCT
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||btrim(dnulot) AS lot_id, -- Identifiant du lot (c'est bon, quand il n'y a pas de lot, c'est la même façon de noter l'identifiant)
btrim(dsgrpf) AS dsgrpf , -- Sous-groupe de nature de culture
btrim(cnatsp) AS cnatsp , -- Code nature de culture spéciale
CASE WHEN trim(dclssf) = '' OR dclssf = '00'
THEN NULL::integer
ELSE ltrim(dclssf,'0')::integer
END AS dclssf, -- Classe dans le groupe et la série tarif
btrim(ccosub) AS ccosub , -- Lettres indicatives de la suf
dcntsf -- Contenance de la suf
FROM "{sch}".suf
WHERE
ccodep||ccocom||replace(ccopre, ' ', '0')||replace(ccosec, ' ', '0')||dnupla||trim(dnulot) IN (SELECT lot_id FROM cadastre.lots)
'''.format(
sch=schema)
if list_parid:
sql += (where_parid(list_parid)
.replace('parcelle.parcelle','suf.suf')
)
with con_fon.begin() as cnx:
cnx.execute(sql)
print('INSERT lotnatcult OK')
def update_typprop(con=con_fon):
sql = '''set work_mem='512MB';
UPDATE
cadastre.parcelles
SET typprop_id = nullif(concat_ws('_',gtoper::text,ccogrm::text),'')
FROM cadastre.proprios
JOIN cadastre.r_prop_cptprop USING (dnuper)
JOIN cadastre.cptprop USING (dnupro)
JOIN cadastre.cadastre USING (dnupro)
JOIN cadastre.lots USING (lot_id)
WHERE lots.par_id = parcelles.par_id
AND typprop_id is NULL
'''
with con.begin() as cnx:
cnx.execute(sql)
print('UPDATE typprop OK')
if __name__ == "__main__":
from pycen import update_to_sql
# par = '3805050000E0523'
par = None
sch = '38_202603'
drop_tables(con_fon)
lst_sch = [
# '07_202603',
'26_202603','42_202603','38_202603']
for sch in lst_sch:
print(' INIT ',sch)
insert_voie(sch)
insert_parcelle(sch)
insert_proprio(sch)
insert_cptprop(sch)
insert_r_prop_cptprop(sch)
insert_lot(sch)
insert_cadastre(sch)
insert_lotnatcult(sch)
cadastre_missing(con_fon)
update_typprop(con_fon)
# pg_restore -h 172.17.0.2 -U postgres --dbname="bd_cen" -c /home/colas/Documents/9_PROJETS/0_FONCIER/DUMP/bd_cen_20240418_16h50_beforemaj2023.dump
_insert_lot1('38_202504','38405000ZD0209')
insert_cadastre('38_202504','380405000ZD0209')
insert_lotnatcult('38_202504','380405000ZD0209')
0_FONCIER/foncier_insert_site.py
Executable
+221
View File
@@ -0,0 +1,221 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Nom : : foncier_insert_table.py
#Description : Insertion des données cadastrales à la base <foncier> après de sa création.
#Copyright : 2021, CEN38
#Auteur : Colas Geier
#Version : 1.0
import pandas as pd
import geopandas as gpd
from sqlalchemy import create_engine
from geoalchemy2 import Geometry, shape
from shapely import wkb
from shapely.geometry.multipolygon import MultiPolygon
from shapely.ops import unary_union
import datetime as dt
import sys
import gc
# Parametres bdd cen_38 #bart (in)
# user_in = 'cen_admin'
# pwd_in = '#CEN38@venir'
# adr_in = '192.168.0.3'
# port_in = '5432'
# base_in = 'bd_cen38'
from pycen import con_bdcen
schema_in = 'sites'
table_in = 'c_sites_zonages'
# Parametres bdd FONCIER NEW (out)
# user_out = 'postgres'
# pwd_out = 'tutu'
# adr_out = '192.168.60.9'
# port_out = '5432'
# base_out = 'bd_cen'
from pycen import con_fon
schema_out = 'sites'
# Parametres généraux
crs = 'EPSG:2154'
chunk = None
# Définition des fonctions
start_time = dt.datetime.today()
def time_exec (init_time):
time = dt.datetime.today() - init_time
return str(time)
def drop_exist(df, con, table, schema, chunk=None):
exist = pd.read_sql_table(
table_name = table,
con = con,
schema = schema,
chunksize = chunk, )
if chunk:
exist = pd.concat(exist, ignore_index=True)
return df
def give_z(x):
if x.type == 'Polygon':
x = [x]
zlist = []
for polygon in x:
zlist.extend([c[-1] for c in polygon.exterior.coords[:-1]])
for inner_ring in polygon.interiors:
zlist.extend([c[-1] for c in inner_ring.coords[:-1]])
return zlist
#return sum(zlist)/len(zlist) #In your case to get mean. Or just return zlist[0] if they are all the same
def get_data(table,schema,engine=con_bdcen,chunk=chunk):
tmp = pd.read_sql_table(
table_name = table,
# sql = sql,
con = engine,
# geom_col = 'geom',
schema = schema,
# crs = crs,
chunksize = chunk, )
# Mise en forme des données
# start_time = dt.datetime.today()
if chunk:
df = pd.concat(tmp, ignore_index=True)
else:
df = tmp.copy()
if 'geom' in df.columns and not df[~df['geom'].isna()].empty:
df = gpd.GeoDataFrame(df)
df['geom'] = df.geom.apply(lambda x: shape.to_shape(x))
df.set_geometry('geom', inplace=True, crs=crs)
return df
# Initiation des connexions
# con_bdcen = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user_in,pwd_in,adr_in,port_in,base_in), echo=False)
# con_in = engine_in.connect()
# engine_out = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user_out,pwd_out,adr_out,port_out,base_out), echo=False)
# con_fon = engine_out.connect()
########################
###### MAIN ######
########################
# get table milieu
tmi = get_data(table = 'd_milieux', schema='sites', engine=con_fon)
# Import data
print('INIT import data ........... %s sec'%( time_exec(start_time) ))
sql = "SELECT * FROM {0}.{1}".format(schema_in, table_in)
df = get_data(table = table_in, schema=schema_in, engine=con_bdcen)
df.loc[(df.code_site=='RNIP')&(df.type_zonage=='ZO'),'nom_site'] = df.loc[(df.code_site=='RNIP')&(df.type_zonage=='ZI'),'nom_site'].values[0]
df = df[df.date_fin.isna()]
df.sort_values('code_site',inplace=True)
# Table sites
tab_out = 'sites'
name_col_out = con_fon.dialect.get_columns(con_fon, tab_out, schema=schema_out)
name_col_out = [ sub['name'] for sub in name_col_out ]
dictio = {
'code_site': 'site_id',
'nom_site' : 'site_nom',
# 'id_mnhn' : 'mnhn_id',
# 'idfcen' : 'fcen_id',
'milieux': 'milieu_id',
'date_ajout': 'annee_creation',
'surface_ha': 'surf_m2',
}
df_sites = gpd.GeoDataFrame(df[dictio.keys()], geometry=df.geom, crs=crs)
df_sites.rename(columns= dictio, inplace=True)
df_sites.rename(columns= {'geometry': 'geom'}, inplace=True)
df_sites.sort_values(['site_id','surf_m2'],inplace=True)
df_sites.drop_duplicates(subset=['site_id','site_nom','surf_m2'],inplace=True)
df_sites.loc[df_sites.site_nom.duplicated(keep='first'),'site_id'] = df_sites.loc[df_sites.site_nom.duplicated(keep='first'),'site_id'] + '_ZO'
df_sites.loc[df_sites.site_nom.duplicated(keep='last'),'site_id'] = df_sites.loc[df_sites.site_nom.duplicated(keep='last'),'site_id'] + '_ZI'
df_sites.loc[df_sites.site_id.str.endswith('ZO'),'site_nom'] = df_sites.loc[df_sites.site_id.str.endswith('ZO'),'site_nom'] + ' - ZO'
df_sites.loc[df_sites.site_id.str.endswith('ZI'),'site_nom'] = df_sites.loc[df_sites.site_id.str.endswith('ZI'),'site_nom'] + ' - ZI'
# Correspondance site / type_site
site_gere = '|'.join(df[df.gestion].code_site.unique().tolist())
site_asst = '|'.join(df[df.assist_cnv|df.assist_col].code_site.unique().tolist())
df_sites['typsite_id'] = '0'
df_sites.loc[df_sites.site_id.str.contains(site_asst), 'typsite_id'] = '2-1'
df_sites.loc[df_sites.site_id.str.contains(site_gere), 'typsite_id'] = '1-1'
# Correspondance site / milieu_lib_simpl
df_sites.loc[df_sites['milieu_id'] == 'Tourbières et marais', 'milieu_id'] = 'Milieux humides'
df_sites.loc[df_sites['milieu_id'] == 'Gîte à chiroptères', 'milieu_id'] = 'Gîtes à chiroptères'
df_sites.loc[df_sites['milieu_id'] == 'Ecosystèmes alluviaux', 'milieu_id'] = 'Milieux alluviaux'
df_sites.loc[df_sites['milieu_id'] == 'Ecosystèmes aquatiques', 'milieu_id'] = 'Milieux aquatiques'
df_sites.loc[df_sites['milieu_id'] == 'Pelouses sèches', 'milieu_id'] = 'Pelouses sèches'
df_sites.loc[df_sites['milieu_id'].isna(), 'milieu_id'] = 'N.P.'
for m in df_sites['milieu_id'].unique():
df_sites.loc[df_sites['milieu_id'] == m, 'milieu_id'] = tmi[
(tmi['milieu_lib_simpl'] == m) |
(tmi['milieu_lib'] == m )
]['milieu_id'].values[0]
# Complétion des champs pour test...
# df_sites['annee_creation'] = '2021'
df_sites.loc[~df_sites.annee_creation.isna(),'annee_creation'] = df_sites.loc[
~df_sites.annee_creation.isna()
,'annee_creation'].astype(str).str[:4]
df_sites['annee_creation'].fillna(9999,inplace=True)
if all(df_sites['geom'].has_z):
# Suppression de la dimension Z
geom_type = df_sites['geom'].geom_type
df_sites['geom'] = [wkb.loads(wkb.dumps(geom, output_dimension=2)) for geom in df_sites['geom']]
df_sites.set_geometry('geom', drop=False, inplace=True, crs=crs)
# df_sites.rename(columns= {'geometry': 'geom'}, inplace=True)
df_sites.to_postgis(
name = tab_out,
con = con_fon,
schema = schema_out,
index = False,
if_exists = 'append',
chunksize = chunk,
geom_col = 'geom',
)
# Table d_typsite
# Table d_milieux
# print('INIT construction data for d_milieux table ...... %s sec'%( time_exec(start_time) ))
# df = pd.DataFrame(DF['tmilieux'])
# df['milieu_lib_simpl'] = None
# df['milieu_descrip'] = None
# df.drop_duplicates(inplace=True)
# df.dropna(inplace=True, how='all',axis=0)
# df = drop_exist(df, con=con_fon, table='d_milieux', schema=schema_out)
# sql = "UPDATE {0}.{1} SET tmilieux = 'Ecosystèmes alluviaux' WHERE tmilieux = 'Ecosystemes alluviaux'".format(schema_in, table_in)
# print(sql)
# con_in.execute(sql)
# for index, d in df.iterrows():
# tmp = df[df.id != d.id].copy()
# tmp['intersect'] = tmp.geometry.intersects(d.geom)
# if not tmp[tmp.intersect].empty:
# print(tmp[tmp.intersect])
# else :
# print('No insterction for {}'.format(d.id))
0_FONCIER/get_infos_parcelle.py
+18
View File
@@ -0,0 +1,18 @@
import pycen
import geopandas as gpd
par_id = '38412000ZD0136'
PATH_OUT = '/home/colas/Documents/tmp/'+par_id
sql = "SELECT par_id, geom FROM cadastre.parcelles_38 WHERE par_id = '%s'"%par_id
parc = gpd.read_postgis(sql,pycen.con_fon)
parc.to_crs(4326).to_file(PATH_OUT+'.geojson', driver='GeoJSON')
parc.to_crs(4326).to_file(PATH_OUT+'.gpkg', driver='GPKG')
sql = "SELECT * FROM zones_humides.v_zoneshumides WHERE ST_INTERSECTS(geom,'SRID=2154;%s')"%parc.geometry[0]
zh =gpd.read_postgis(sql,pycen.con)
sql = "SELECT * FROM saisie.vm_synthese_observations WHERE ST_INTERSECTS(geom,'SRID=2154;%s')"%parc.geometry[0]
obs_sicen =gpd.read_postgis(sql,pycen.con_sicen)
0_FONCIER/grant_table.py
+52
View File
@@ -0,0 +1,52 @@
from pycen import con_bdcen,con_cad,con_fon as con
import geopandas as gpd
from sqlalchemy import create_engine, text
def get_list_table(con,sch):
return con.dialect.get_table_names(con,schema=sch)
def get_list_view(con,sch):
return con.dialect.get_view_names(con,schema=sch)
def get_table_pkey(con,sch,tab):
return con.dialect.get_foreign_keys(con,tab,sch)['constrained_columns']
def grant_all_table(con,sch):
# sch = 'cadastre'
sql = 'GRANT USAGE ON SCHEMA "%s" TO grp_consult;'%sch
with con.begin() as cnx:
cnx.execute(sql)
print('GRANT USAGE TO grp_consult FOR SCHEMA "%s"'%(sch))
for tab in get_list_table(con,sch):
sql = 'GRANT SELECT ON TABLE "%s".%s TO grp_consult;'%(sch,tab)
with con.begin() as cnx:
cnx.execute(sql)
print('GRANT SELECT TO grp_consult FOR "%s".%s'%(sch,tab))
for tab in get_list_view(con,sch):
sql = 'GRANT SELECT ON TABLE "%s".%s TO grp_consult;'%(sch,tab)
with con.begin() as cnx:
cnx.execute(sql)
print('GRANT SELECT TO grp_consult FOR "%s".%s'%(sch,tab))
def revoke_all_table(con,sch):
for tab in get_list_table(con,sch):
sql = 'REVOKE SELECT ON "%s".%s FROM grp_consult;'%(sch,tab)
with con.begin() as cnx:
cnx.execute(sql)
print('REVOKE SELECT grp_consult FOR "%s".%s'%(sch,tab))
for tab in get_list_view(con,sch):
sql = 'REVOKE SELECT ON "%s".%s FROM grp_consult;'%(sch,tab)
with con.begin() as cnx:
cnx.execute(sql)
print('REVOKE SELECT TO grp_consult FOR "%s".%s'%(sch,tab))
if __name__ == "__main__":
sch_cad = '42_202603'
sch_old = '42_202501'
grant_all_table(con_cad,sch_cad)
# revoke_all_table(con_cad,sch_old)
0_FONCIER/new_sites.py
+36
View File
@@ -0,0 +1,36 @@
from pycen import con_fon, update_to_sql
import geopandas as gpd
from os import path
def milieu(x):
d_milieu = gpd.pd.read_sql('SELECT * FROM sites.d_milieux',con_fon)
dict_m = dict(zip(d_milieu.milieu_lib,d_milieu.milieu_id))
x.replace({'milieu':dict_m},inplace=True)
x.rename(columns={'milieu':'milieu_id'},inplace=True)
def typ_site(x):
d_site = gpd.pd.read_sql('SELECT * FROM sites.d_typsite',con_fon)
dict_s = dict(zip(d_site.typsite_lib,d_site.typsite_id))
x.replace({'typ_site':dict_s},inplace=True)
x.rename(columns={'typ_site':'typsite_id'},inplace=True)
if __name__ == "__main__":
PATH = '/home/colas/Documents/tmp/Foncier/LONE'
file = 'LONE.xlsx'
geofile = 'Contour_site_lônes.shp'
df = (gpd.pd.read_excel(path.join(PATH,file))
.rename(columns={'id_site':'site_id'}))
milieu(df)
typ_site(df)
_gdf = gpd.read_file(path.join(PATH,geofile))
gdf = gpd.GeoDataFrame(df,geometry=[_gdf.unary_union],crs=_gdf.crs)
gdf.rename_geometry('geom',inplace=True)
gdf['surf_m2'] = gdf.area
gdf.to_postgis('sites',con_fon,'sites',if_exists='append',index=False)
gdf.rename_geometry('geom_ecolo',inplace=True)
update_to_sql(gdf,con_fon,'sites','sites','site_id')
0_FONCIER/remove_parcelles_site.py
+39
View File
@@ -0,0 +1,39 @@
from pycen import con_fon
import geopandas as gpd
import sys
site_id = 'CRAS'
lst_parcelle = None
site = gpd.read_postgis(
"SELECT * FROM sites.v_sites WHERE site_id = '%s'"%site_id,
con_fon)
parc = gpd.read_postgis(
"SELECT * FROM sites.v_sites_parcelles WHERE site_id = '%s'"%site_id,
con_fon)
# lst_parcelle = parc[~parc.intersects(site.unary_union)].par_id.unique().tolist()
lst_parcelle = parc.par_id.unique().tolist()
for par_id in lst_parcelle:
try:
with con_fon.begin() as cnx:
cnx.execute("SELECT cadastre.suppr_parcelles_cen('{par_id}','{site_id}')"\
.format(
par_id=par_id,site_id=site_id
)
)
except Exception as e:
print(e)
print('\n')
print('La parcelle %s ne peut pas être supprimé'%par_id)
with con_fon.begin() as cnx:
r = cnx.execute("SELECT admin_sig.refresh_mview_foncier('{site_id}')"\
.format(
site_id=site_id
)
)
res = r.fetchall()
0_FONCIER/v_veillefonciere.py
+27
View File
@@ -0,0 +1,27 @@
from sqlalchemy import text
from pycen import con_fon
v_veillefonciere = """
DROP VIEW IF EXISTS _tdb_f6cen.v_veillefonciere;
CREATE OR REPLACE VIEW _tdb_f6cen.v_veillefonciere
AS
SELECT
row_number() OVER (ORDER BY SPLIT_PART(site_id,'_',1) ASC) AS gid,
SPLIT_PART(site_id,'_',1) AS site_id,
SPLIT_PART(site_nom,' - Z',1) AS site_nom,
enjeu_lib,
'CONSERVATOIRE D''ESPACES NATURELS ISERE'::text AS cen,
38::int AS dept,
st_area(st_union(geom_vf)) AS superficie,
st_union(geom_vf) geom
FROM sites.sites
JOIN sites.d_enjeux USING (enjeu_id)
WHERE veillefonciere
GROUP BY 2,3,4
ORDER BY 1
;
"""
with con_fon.begin() as cnx:
cnx.execute(text(v_veillefonciere))