374 lines
12 KiB
Python
374 lines
12 KiB
Python
#!/usr/bin/env python
|
|
# -*- coding: UTF-8 -*-
|
|
|
|
from pycen import con_bdcen, con_cad
|
|
from os import getlogin
|
|
from os.path import join
|
|
from sqlalchemy.engine import URL
|
|
from sqlalchemy import create_engine
|
|
import geopandas as gpd
|
|
import pandas as pd
|
|
|
|
user = 'postgres'
|
|
pwd = 'postgres'
|
|
adr = '172.17.0.2'
|
|
base = 'postgres'
|
|
url = URL.create('postgresql+psycopg2',
|
|
username=user,
|
|
password=pwd,
|
|
host=adr,
|
|
database=base,
|
|
)
|
|
con_vm = create_engine(url)
|
|
|
|
annee = 2023
|
|
|
|
PATH = '/media/{login}/SRV/FICHIERS/OUTILS/BASES DE DONNEES/BILAN_FEDE_CEN/{an}/'
|
|
path_source = join(PATH,'Docs_SIG_joints{an}/').format(login=getlogin(),an=annee)
|
|
dico_data = 'Dico_DATA_sites_CEN_v%i_v1.xlsx' % annee
|
|
sheet_par = 'parcelles_cen_xx_%i' % annee
|
|
sheet_sit = 'sites_cen_xx_%i' % annee
|
|
|
|
# lv_cen = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='A:D',nrows=25)
|
|
# lv_typmaitr = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='F:K',nrows=23)
|
|
# lv_typmilie = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='M:N',nrows=16)
|
|
# lv_echelnum = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='P:Q',nrows=8)
|
|
# lv_typhab = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=13, usecols='P:Q',nrows=5)
|
|
# lv_sourcegeom = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=1, usecols='S:U',nrows=12)
|
|
# lv_lienres = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=20, usecols='M:N',nrows=4)
|
|
# lv_typprop = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=45, usecols='A:H',nrows=12)
|
|
# lv_codgeol = pd.read_excel(path_source+dico_data,sheet_name='listes_valeurs (copy&modif)',header=36, usecols='M:T',nrows=19)
|
|
lst_cols_parc = pd.read_excel(path_source+dico_data,sheet_name=sheet_par,header=0, usecols='F',nrows=26)\
|
|
['nom du champ'].tolist()
|
|
|
|
lst_prop = ['Acquis par le CEN ou CREN','ACQUIS&CONVENTION', 'PARTIELLEMENT MAITRISE','ACQUIS PAR LE CEN EN BND','ACQUIS PAR LE CEN ISERE']
|
|
lst_conv = ["CONVENTION D'USAGE",'CONVENTION']
|
|
lst_gere = ['BAIL EMPHYTEOTIQUE', *lst_conv, *lst_prop]
|
|
|
|
DICT_COLS = {
|
|
'id' : 'id_site_cen_parc',
|
|
'par_id' : 'num_parcelle',
|
|
'section': 'num_section',
|
|
'code_dep':'insee_dep',
|
|
'code_com':'insee_com',
|
|
|
|
}
|
|
|
|
def update_idproprietaire(df):
|
|
if 'id_proprietaire' in df.columns :
|
|
df['id_proprietaire'] = 0
|
|
|
|
df.loc[df.classif_prop.str.contains('CEN RA'),'id_proprietaire'] = 28
|
|
df.loc[df.classif_prop.str.contains('CEN ISERE'),'id_proprietaire'] = 14
|
|
df.id_proprietaire.fillna(0,inplace=True)
|
|
|
|
return df
|
|
|
|
|
|
|
|
def update_typeprop(df):
|
|
|
|
DICT = {
|
|
'Etat':'2_1_PUB',
|
|
'CEN RA - BND':'3_CEN',
|
|
'CEN RA':'3_CEN',
|
|
'CEN ISERE - BND':'3_CEN',
|
|
'Privé et entreprises':'1_PRI',
|
|
'Communes et comcom':'2_4_COM',
|
|
'Autres public et assimiliés (syndicats)':'2_0_MIX',
|
|
'Département':'2_3_PUB',
|
|
'Autres asso et fédérations':'2_6_PRI',
|
|
'FRUP':'3_CEN',
|
|
}
|
|
|
|
df['type_prop'] = df['classif_prop'].copy()
|
|
df['type_prop'].replace(DICT, inplace=True)
|
|
|
|
df.loc[
|
|
(df.classif_prop=='Privé et entreprises') &
|
|
(df.gtoper.str.contains('2')),
|
|
'type_prop'] = '2_0_MIX'
|
|
df.loc[
|
|
df.ccogrm == 9,
|
|
'type_prop'] = '2_9_PUB'
|
|
|
|
return df
|
|
|
|
|
|
def update_codemfu(df):
|
|
|
|
df.loc[df.maitrise.isin(lst_prop),'code_mfu1'] = 'P'
|
|
df.loc[df.maitrise.isin(lst_conv),'code_mfu1'] = 'C'
|
|
df.loc[df.maitrise == 'BAIL EMPHYTEOTIQUE','code_mfu1'] = 'L'
|
|
df.loc[df.maitrise.isin(lst_prop),'code_mfu2'] = 'P1'
|
|
df.loc[df.maitrise.isin(lst_conv),'code_mfu2'] = 'C7'
|
|
df.loc[df.maitrise == 'BAIL EMPHYTEOTIQUE','code_mfu2'] = 'L1'
|
|
|
|
df.loc[df.maitrise == 'ACCORD VERBAL','code_mfu1'] = 'C'
|
|
df.loc[df.maitrise == 'ACCORD VERBAL','code_mfu2'] = 'C17'
|
|
|
|
df.loc[df.maitrise == 'CONVENTION','code_mfu1'] = 'C'
|
|
df.loc[df.maitrise == 'CONVENTION','code_mfu2'] = 'C7'
|
|
|
|
df.loc[df.indivision=='USUFRUIT','code_mfu2'] = 'P2'
|
|
|
|
return df
|
|
|
|
|
|
def update_bnd(df):
|
|
|
|
df.loc[df.maitrise == 'PARTIELLEMENT MAITRISE','bnd'] = 1
|
|
df.loc[df.maitrise != 'PARTIELLEMENT MAITRISE','bnd'] = 0
|
|
df.loc[df.maitrise == 'ACQUIS&CONVENTION','bnd'] = 1
|
|
|
|
return df
|
|
|
|
|
|
def get_parc_milli():
|
|
|
|
sqlsit = '''
|
|
SELECT * FROM %s.%s
|
|
WHERE (date_fin is NULL OR date_fin >= '%i-01-01') AND type_site='MILI' ;'''%('sites','c_sites_zonages',annee)
|
|
df1 = gpd.read_postgis(sql=sqlsit,geom_col='geom',con=con_bdcen)
|
|
|
|
sql0 = '''SELECT
|
|
t1.geo_parcelle, substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) par_id,
|
|
t1.annee, t1.update_dat, t4.ccopre, t4.ccosec, t4.dcntpa, t1.geom, STRING_AGG(t3.gtoper::text,',') gtoper, STRING_AGG(t3.ccogrm::text,',') ccogrm,
|
|
STRING_AGG(CONCAT(t3.gtoper::text||COALESCE('_'||t3.ccogrm::text,'')),',') type_prop, STRING_AGG(t3.ddenom::text,',') ddenom
|
|
FROM "{sch}"."{t1}" t1
|
|
LEFT JOIN ("{sch}".parcelle_info t2
|
|
LEFT JOIN "{sch}".proprietaire t3 USING (comptecommunal))
|
|
USING (geo_parcelle)
|
|
LEFT JOIN "{sch}".parcelle t4 ON t1.geo_parcelle = t4.parcelle
|
|
'''.format(
|
|
sch='38_%i07'%(annee-1),
|
|
t1='v_geo_parcelle')
|
|
sql1 = '''
|
|
WHERE ST_INTERSECTS(t1.geom,'SRID={epsg};{poly}')
|
|
GROUP BY 1,2,3,4,5,6,7,8
|
|
'''.format(epsg=2154,poly=df1.unary_union.buffer(-1))
|
|
sql = sql0 + sql1
|
|
df2 = gpd.read_postgis(sql,con_cad)
|
|
df2['code_dep'] = df2.par_id.str[:2]
|
|
df2['code_com'] = df2.par_id.str[2:5]
|
|
df2['code_mfu1'] = 'C'
|
|
df2['code_mfu2'] = 'C12'
|
|
df2['section'] = df2.ccosec.copy()
|
|
df2['id_site_cen_parc'] = 0
|
|
df2['bnd'] = 0
|
|
df2['mesure_compens'] = 0
|
|
df2['pour_part'] = 0
|
|
|
|
return df2
|
|
|
|
|
|
def update_mc(df):
|
|
|
|
sqlsit = '''
|
|
SELECT * FROM %s.%s
|
|
WHERE (date_fin is NULL OR date_fin >= '%i-01-01') AND type_site='MC' ;'''%('sites','c_sites_zonages',annee)
|
|
df1 = gpd.read_postgis(sql=sqlsit,geom_col='geom',con=con_bdcen)
|
|
|
|
is_mc = df.intersects(df1.unary_union)
|
|
df['mesure_compens'] = 0
|
|
df.loc[
|
|
(is_mc & df.maitrise.isin(lst_conv)
|
|
)|(df.Parc_MCE=='OUI'),
|
|
'mesure_compens'
|
|
] = 1
|
|
|
|
return df
|
|
|
|
|
|
def update_pourpart(df):
|
|
|
|
df['pour_part'] = 0
|
|
df.loc[
|
|
df.maitrise=='PARTIELLEMENT MAITRISE',
|
|
'pour_part'
|
|
] = 1
|
|
|
|
return df
|
|
|
|
|
|
def is_rnx(df):
|
|
|
|
sqlsit = '''
|
|
SELECT * FROM %s.%s
|
|
WHERE (date_fin is NULL OR date_fin >= '%i-01-01') AND type_site='RNN' ;'''%('sites','c_sites_zonages',annee)
|
|
df1 = gpd.read_postgis(sql=sqlsit,geom_col='geom',con=con_bdcen)
|
|
|
|
isrnx = df.intersects(df1.unary_union.buffer(-1))
|
|
df['parc_gestion_rnx'] = 0
|
|
df.loc[isrnx,'parc_gestion_rnx'] = 1
|
|
|
|
return df
|
|
|
|
|
|
def update_datemajparcelle(df,cad):
|
|
|
|
tmp = df.merge(
|
|
parcx[['par_id','update_dat']],
|
|
how='left',
|
|
left_on='num_parcelle',
|
|
right_on='par_id')
|
|
tmp.rename(columns={'update_dat':'date_maj_parcelle'},inplace=True)
|
|
|
|
return tmp
|
|
|
|
|
|
def modif_FRUP2023(df):
|
|
|
|
df = df[
|
|
~df.commentair.str.contains(
|
|
# 'sign&2023|acq&2023',
|
|
r'sign.*2023|acq.*2023',
|
|
na=False,
|
|
case=False,
|
|
regex=True)]
|
|
is_frup = df.maitrise=='FRUP'
|
|
df.loc[is_frup,'classif_prop'] = 'CEN ISERE'
|
|
df.loc[is_frup,'maitrise'] = 'ACQUIS PAR LE CEN ISERE'
|
|
|
|
return df
|
|
|
|
|
|
def complete_tab(df):
|
|
|
|
df['id_site_cen_parc'] = df['id_site_cen_parc'].astype(int)
|
|
df['id_site_fcen_parc'] = None
|
|
df['bnd'] = df['bnd'].astype(int)
|
|
df.loc[df.type_prop.str.contains(','),'type_prop'] = '2_0'
|
|
|
|
df['surf_ore_m2'] = 0
|
|
df['date_debut_ore'] = None
|
|
df['date_fin_ore'] = None
|
|
df['doc_foncier'] = 1
|
|
df['source_doc_foncier'] = 0
|
|
df['source_geom_parc_nature'] = 2
|
|
df['source_geom_parc_annee'] = annee - 1
|
|
df['echelle_num_parc'] = 0
|
|
df['source_surf_parc'] = 1
|
|
df = update_datemajparcelle(df,parcx)
|
|
df['domaine_public'] = 0
|
|
|
|
return df
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
shplastsend = gpd.read_file(
|
|
join(PATH.format(login=getlogin(),an=annee-1),'ENVOIE','Parcelles_CEN_38_2022.shp')
|
|
)
|
|
lastsend = pd.read_csv(PATH.format(login=getlogin(),an=annee-1)+'ENVOIE/Parcelles_CEN_38_2022.csv')
|
|
|
|
sql = '''
|
|
SELECT * FROM foncier.c_anim_foncier_V2
|
|
WHERE maitrise not in ('Acquis par le CDI','Acquis par la commune ou Comcom','CONVENTION DE PASSAGE') AND maitrise is not NULL
|
|
; '''
|
|
dfparc = gpd.read_postgis(sql,con_bdcen)
|
|
|
|
if annee == 2023:
|
|
dfparc = modif_FRUP2023(dfparc)
|
|
|
|
dfparc = update_codemfu(dfparc)
|
|
dfparc = update_bnd(dfparc)
|
|
dfparc = update_pourpart(dfparc)
|
|
dfparc = update_mc(dfparc)
|
|
dfparc = is_rnx(dfparc)
|
|
dfparc = update_idproprietaire(dfparc)
|
|
dfparc = update_typeprop(dfparc)
|
|
|
|
|
|
dfparc['surf_parc_maitrise_m2'] = dfparc.area
|
|
|
|
# COLUMNS :
|
|
# Index(['par_id', 'geom', 'dcntpa', 'typprop_id', 'ccopre', 'section', 'dnupla',
|
|
# 'nom_com', 'numero', 'code_dep', 'code_com', 'gtoper', 'ccogrm',
|
|
# 'type_prop', 'nom_proprio', 'id', 'Parc_MCE', 'docs_off_num',
|
|
# 'Nbr_lots_BND', 'partiel_conv_ou_acq', 'code_mfu1', 'code_mfu2', 'bnd',
|
|
# 'mesure_compens', 'pour_part', 'par_id_v2', 'maitrise', 'date_der_c',
|
|
# 'indivision', 'contact', 'avis_propr', 'commentair', 'nom_proprio_old',
|
|
# 'site_id', 'type_zone', 'last_upd_parc', 'classif_prop', 'date_debut',
|
|
# 'date_fin'],
|
|
# dtype='object')
|
|
|
|
#
|
|
tmp = dfparc.rename(columns=DICT_COLS).columns.isin(lst_cols_parc)
|
|
lst_col = dfparc.rename(columns=DICT_COLS).columns[tmp]
|
|
parc = dfparc.rename(columns=DICT_COLS)[lst_col].copy()
|
|
shp_parc = dfparc[['par_id','geom']].rename(columns=DICT_COLS).copy()
|
|
|
|
|
|
sql0 = '''SELECT
|
|
t1.geo_parcelle, substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) par_id,
|
|
t1.annee, t1.update_dat, t4.ccopre, t4.ccosec, t4.dcntpa, t1.geom, STRING_AGG(t3.gtoper::text,',') gtoper, STRING_AGG(t3.ccogrm::text,',') ccogrm,
|
|
STRING_AGG(CONCAT(t3.gtoper::text||COALESCE('_'||t3.ccogrm::text,'')),',') type_prop, STRING_AGG(t3.ddenom::text,',') ddenom
|
|
FROM "{sch}"."{t1}" t1
|
|
LEFT JOIN ("{sch}".parcelle_info t2
|
|
LEFT JOIN "{sch}".proprietaire t3 USING (comptecommunal))
|
|
USING (geo_parcelle)
|
|
LEFT JOIN "{sch}".parcelle t4 ON t1.geo_parcelle = t4.parcelle
|
|
'''.format(
|
|
sch='38_202207',
|
|
t1='v_geo_parcelle')
|
|
sql1 = '''WHERE substring(t1.geo_parcelle from 1 for 2)||substring(t1.geo_parcelle from 4 for 12) IN ('%s')
|
|
GROUP BY 1,2,3,4,5,6,7,8'''%"','".join(dfparc.par_id.tolist())
|
|
sql = sql0 + sql1
|
|
|
|
parcx = gpd.read_postgis(sql,con=con_cad)
|
|
noparcx = dfparc[~dfparc.par_id.isin(parcx.par_id)]
|
|
|
|
newparc = parc[~parc.num_parcelle.isin(lastsend.num_parcelle)].copy()
|
|
shp_newparc = shp_parc[~shp_parc.num_parcelle.isin(lastsend.num_parcelle)].copy()
|
|
newparcx = parcx[parcx.par_id.isin(shp_newparc.num_parcelle)].copy()
|
|
shp_newparc = shp_newparc.merge(newparcx[['par_id','update_dat']],how='left',left_on='num_parcelle',right_on='par_id')\
|
|
.drop(columns=['par_id'])\
|
|
.rename(columns={
|
|
'num_parcelle':'num_parc',
|
|
'update_dat':'date_maj_p'
|
|
})\
|
|
.rename_geometry('geometry')
|
|
|
|
|
|
newparc = complete_tab(newparc)
|
|
parc = complete_tab(parc)
|
|
# mili = get_parc_milli()
|
|
|
|
PARC = pd.concat([lastsend,newparc])
|
|
PARC.id_site_cen_parc = PARC.id_site_cen_parc.astype(int)
|
|
|
|
|
|
# Mise au format des dates
|
|
date_cols = PARC.columns[PARC.columns.str.contains('date')]
|
|
for c in date_cols:
|
|
PARC[c] = pd.to_datetime(PARC[c],yearfirst=True).dt.strftime('%Y/%m/%d')
|
|
# site2023[c] = site2023[c].dt.strftime('%Y/%m/%d')
|
|
|
|
|
|
|
|
|
|
SHP_PARC = gpd.GeoDataFrame(
|
|
pd.concat([shplastsend,shp_newparc]),
|
|
geometry='geometry',
|
|
crs=2154
|
|
)
|
|
SHP_PARC['date_maj_p'] = pd.to_datetime(SHP_PARC['date_maj_p'],yearfirst=True).dt.strftime('%Y/%m/%d')
|
|
SHP_PARC.geometry = SHP_PARC.buffer(0)
|
|
|
|
PARC.to_csv(
|
|
join(
|
|
PATH.format(login=getlogin(),an=annee),
|
|
'Recueil_data',
|
|
'parcelles_cen_38_2023.csv'),
|
|
index=False)
|
|
SHP_PARC.to_file(
|
|
join(
|
|
PATH.format(login=getlogin(),an=annee),
|
|
'Recueil_data',
|
|
'parcelles_cen_38_2023.shp'),
|
|
index=False)
|
|
|
|
|