init depot renew
This commit is contained in:
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: UTF-8 -*-
|
||||
from pycen import con_sicen, update_to_sql
|
||||
import pandas as pd
|
||||
import geopandas as gpd
|
||||
from os import path
|
||||
|
||||
def get_site_serena():
|
||||
sql = '''
|
||||
SELECT s.*,
|
||||
t.*,
|
||||
c.choi_nom
|
||||
FROM serenabase.rnf_site s
|
||||
JOIN serenabase.tmp_sgll t ON t.sgll_site_id = s.site_id
|
||||
JOIN serenabase.rnf_choi c ON c.choi_id = s.site_categ_choi_id
|
||||
'''
|
||||
return pd.read_sql_query(sql,con_sicen).dropna(axis=1,how='all')
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
# Lecture des sites SIG
|
||||
PATH = '/media/cgeier/SRV/FICHIERS/OUTILS/BASES DE DONNEES/RHOMEO/2018/RhoMéO_Données'
|
||||
FILE = '38_sig_points_de_suivis_point.shp'
|
||||
sig = gpd.read_file(path.join(PATH,FILE))
|
||||
|
||||
# Récupération des sites SERENA
|
||||
site_serena = get_site_serena()
|
||||
no_coord = site_serena.sgll_lat=='999'
|
||||
is_rhomeo = site_serena.site_nom.str[-3:].str.isnumeric()
|
||||
is_rhomeo2 = site_serena.choi_nom.str.contains('rhomeo',case=False)
|
||||
# site_serena[no_coord]
|
||||
# site_serena[is_rhomeo]
|
||||
|
||||
# Isolation des sites RHOMEO
|
||||
site_rhomeo_all = site_serena[is_rhomeo2&no_coord].copy()
|
||||
site_rhomeo = site_rhomeo_all.merge(sig[['id_regio','geometry']],left_on='site_ref_sig',right_on='id_regio')\
|
||||
.set_geometry('geometry',crs=2154)
|
||||
site_rhomeo.to_crs(4326,inplace=True)
|
||||
site_rhomeo.sgll_lon = site_rhomeo.geometry.x.astype(str)
|
||||
site_rhomeo.sgll_lat = site_rhomeo.geometry.y.astype(str)
|
||||
updt_site = site_rhomeo[['sgll_site_id','sgll_lat','sgll_lon']].copy()
|
||||
|
||||
# Update data
|
||||
update_to_sql(
|
||||
df=updt_site,
|
||||
con=con_sicen,
|
||||
table_name='tmp_sgll',
|
||||
schema_name='serenabase',
|
||||
key_name='sgll_site_id'
|
||||
)
|
||||
@@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: UTF-8 -*-
|
||||
import pandas_access as mdb # pour lecture de la bd medwet
|
||||
from os import path
|
||||
from pycen import con_sicen, update_to_sql
|
||||
import pandas as pd
|
||||
|
||||
# access / postgis
|
||||
SOURCE = 'postgis'
|
||||
postgis_sch = 'serenabase'
|
||||
|
||||
DICT_RLV_COLS = {
|
||||
'relv_id':'id_etude',
|
||||
'relv_nom':'nom_etude',
|
||||
'relv_prop_libel':'proprietaire',
|
||||
'relv_1date_c':'date_debut',
|
||||
'relv_2date_c':'date_fin',
|
||||
'relv_comment':'description',
|
||||
}
|
||||
|
||||
DICT_OBS_COLS = {
|
||||
'obse_id':'id_origine',
|
||||
'obse_relv_id':'id_etude',
|
||||
}
|
||||
|
||||
|
||||
def list_access_datatable():
|
||||
|
||||
PATH_IN = '/media/cgeier/SRV/FICHIERS/OUTILS/BASES DE DONNEES/ESPECES/backup'
|
||||
BDD_IN = 'AVENIR.serena_V2.mdb'
|
||||
|
||||
LST_SCHEMA = mdb.read_schema(path.join(PATH_IN,BDD_IN))
|
||||
LST_TABLE = [ *LST_SCHEMA.keys() ]
|
||||
LST_TABLE = mdb.list_tables(path.join(PATH_IN,BDD_IN))
|
||||
TABLE_DATA= []
|
||||
|
||||
for TABLE in LST_TABLE:
|
||||
DTYPE = {x : float for x in LST_SCHEMA[TABLE].keys() if LST_SCHEMA[TABLE][x]=='Long Integer'}
|
||||
df = mdb.read_table(path.join(PATH_IN,BDD_IN),TABLE, dtype=DTYPE)
|
||||
if df.shape[0] == 0:
|
||||
print('%s IS EMPTY !'%TABLE)
|
||||
else :
|
||||
TABLE_DATA += [TABLE]
|
||||
return TABLE_DATA
|
||||
|
||||
|
||||
def list_releve(source='postgis'):
|
||||
if source=='postgis':
|
||||
|
||||
choi = pd.read_sql_table('rnf_choi',con_sicen,postgis_sch)\
|
||||
.dropna(axis=1,how='all')
|
||||
relv = pd.read_sql_table('rnf_relv',con_sicen,postgis_sch)\
|
||||
.dropna(axis=1,how='all')
|
||||
|
||||
# Correction coquille
|
||||
relv.loc[relv.relv_categ_choi_id==100800,'relv_categ_choi_id'] = 100008
|
||||
|
||||
# Jointure relevé / catégorie
|
||||
df = relv.merge(choi[['choi_id','choi_nom']],how='inner',left_on='relv_categ_choi_id',right_on='choi_id')\
|
||||
.drop(columns=['relv_categ_choi_id'])
|
||||
|
||||
# Mise en forme des dates
|
||||
lstcol_dat = df.columns[df.columns.str.contains('date')]
|
||||
for lcd in lstcol_dat:
|
||||
# df[lcd] = df[lcd].replace({'VIDE':None})
|
||||
df.loc[df[lcd]=='2000 à 2001',lcd] = '2000' if '1date' in lcd else '2001'
|
||||
df[lcd].replace({'VIDE':None},inplace=True)
|
||||
df[lcd] = pd.to_datetime(df[lcd])
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def get_serena_obs(source='postgis'):
|
||||
if source=='postgis':
|
||||
obs = pd.read_sql_table('rnf_obse',con_sicen,postgis_sch)
|
||||
|
||||
return obs
|
||||
|
||||
|
||||
|
||||
def get_sicen_obs():
|
||||
return pd.read_sql_table('saisie_observation',con_sicen,'saisie')
|
||||
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
|
||||
if SOURCE == 'access':
|
||||
lst_tab = list_access_datatable()
|
||||
elif SOURCE == 'postgis':
|
||||
lstrelv_serena = list_releve()
|
||||
obs_serena = get_serena_obs()
|
||||
obs_serena = obs_serena.merge(lstrelv_serena[['relv_id','relv_nom']],left_on='obse_relv_id',right_on='relv_id')\
|
||||
.drop(columns='relv_id')
|
||||
|
||||
# Isolation des relevés HORS RhoMéo
|
||||
is_relv = [x for x in lstrelv_serena.relv_nom.str[:3].unique() if not x.isnumeric()]
|
||||
relv_serena = lstrelv_serena[lstrelv_serena.relv_nom.str[:3].isin(is_relv)]
|
||||
|
||||
# Formatage des relevés HORS RhoMéo
|
||||
relv_2sicen = relv_serena.rename(columns=DICT_RLV_COLS).filter(DICT_RLV_COLS.values(),axis=1)
|
||||
relv_2sicen.sort_values('id_etude',inplace=True)
|
||||
relv_2sicen.reset_index(inplace=True,drop=True)
|
||||
relv_2sicen.set_index(-relv_2sicen.index,inplace=True)
|
||||
|
||||
|
||||
# # Intégration des relevés SERENA dans la table 'etude'
|
||||
# relv_2sicen.drop(columns=['id_etude','proprietaire'])\
|
||||
# .rename_axis('id_etude')\
|
||||
# .to_sql('etude',con_sicen,'md',if_exists='append')
|
||||
|
||||
|
||||
# Récupération des données SICEN2
|
||||
# id_lot = 'SERENA'
|
||||
sicen_obs = get_sicen_obs()
|
||||
is_serena = sicen_obs.id_lot==3
|
||||
is_horsetude = sicen_obs.id_etude==2
|
||||
lst_idorigine = sicen_obs[is_serena].id_origine.astype(int).tolist()
|
||||
|
||||
# isolation des données "suposées" non présentes
|
||||
obs_notinsicen = obs_serena[~obs_serena.obse_id.isin(lst_idorigine)].copy()
|
||||
# rhomeo_notinsicen = obs_notinsicen.obse_nom.str.startswith('RhoMéO',na=False)
|
||||
# obs_notinsicen[rhomeo_notinsicen]
|
||||
# test = obs_notinsicen.obse_site_id == 101335
|
||||
# obs_notinsicen[test]
|
||||
|
||||
sql1 = "SELECT * FROM {sch}.tmp_ogll WHERE ogll_obse_id IN {lst} AND ogll_lat <> '999'"
|
||||
res1 = pd.read_sql_query(sql1.format(sch=postgis_sch,lst=tuple(obs_notinsicen.obse_id)),con_sicen)
|
||||
sql2 = "SELECT * FROM {sch}.tmp_sgll WHERE sgll_site_id IN {lst} AND sgll_lat <> '999'"
|
||||
res2 = pd.read_sql_query(sql2.format(sch=postgis_sch,lst=tuple(obs_notinsicen.obse_site_id.unique())),con_sicen)
|
||||
|
||||
|
||||
miss_serena1 = obs_notinsicen.merge(
|
||||
res1.rename(columns={
|
||||
'ogll_obse_id':'obse_id',
|
||||
'ogll_lat':'lat',
|
||||
'ogll_lon':'lon'
|
||||
}),
|
||||
on='obse_id')
|
||||
miss_serena2 = obs_notinsicen.merge(
|
||||
res2.rename(columns={
|
||||
'sgll_site_id':'obse_site_id',
|
||||
'sgll_lat':'lat',
|
||||
'sgll_lon':'lon'
|
||||
}),
|
||||
on='obse_site_id')
|
||||
miss_serena = pd.concat([miss_serena1,miss_serena2])
|
||||
rhomeo_missserena = miss_serena.obse_nom.str.startswith('RhoMéO',na=False)
|
||||
miss_serena[rhomeo_missserena]
|
||||
|
||||
# Formatages des données OBS
|
||||
obs_2sicen = obs_serena[obs_serena.obse_relv_id.isin(relv_serena.relv_id)].rename(columns=DICT_OBS_COLS).filter(DICT_OBS_COLS.values(),axis=1)
|
||||
obs_2sicen['id_lot'] = 3
|
||||
obs_2sicen.id_origine = obs_2sicen.id_origine.astype(str)
|
||||
|
||||
# Correspondance id_relv (SERENA) / id_etude (SICEN2)
|
||||
DICT_ID_ETUDE = dict(zip(relv_2sicen.id_etude,relv_2sicen.index))
|
||||
obs_2sicen.id_etude.replace(DICT_ID_ETUDE,inplace=True)
|
||||
|
||||
# Mise à jour du champs id_etude de la table saisie_observation
|
||||
update_to_sql(
|
||||
obs_2sicen[obs_2sicen.id_origine.astype(int).isin(lst_idorigine)],
|
||||
con_sicen,
|
||||
'saisie_observation',
|
||||
'saisie',
|
||||
['id_lot','id_origine']
|
||||
)
|
||||
|
||||
|
||||
# identification des données restantes taguées "hors etude"
|
||||
id_origine_he = sicen_obs[is_serena&is_horsetude].id_origine.astype(int)
|
||||
is_horsetude2 = obs_serena.obse_id.isin(id_origine_he)
|
||||
res_he = obs_serena[is_horsetude2].copy()
|
||||
DICT2_OBS_COLS = {**DICT_OBS_COLS,'relv_nom':'id_waypoint'}
|
||||
obs2_2sicen = res_he.rename(columns=DICT2_OBS_COLS).filter(DICT2_OBS_COLS.values(),axis=1)
|
||||
obs2_2sicen['id_lot'] = 3
|
||||
obs2_2sicen['id_etude'] = -152
|
||||
obs2_2sicen.id_origine = obs2_2sicen.id_origine.astype(str)
|
||||
|
||||
# Mise à jour du champs id_etude de la table saisie_observation
|
||||
update_to_sql(
|
||||
obs2_2sicen,
|
||||
con_sicen,
|
||||
'saisie_observation',
|
||||
'saisie',
|
||||
['id_lot','id_origine']
|
||||
)
|
||||
|
||||
@@ -0,0 +1,464 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: UTF-8 -*-
|
||||
import pandas_access as mdb
|
||||
from pycen import con_sicen, update_to_sql
|
||||
import pandas as pd
|
||||
import geopandas as gpd
|
||||
from os import path
|
||||
|
||||
|
||||
postgis_sch = 'serenabase'
|
||||
ACCESS_PATH = '/media/cgeier/SRV/FICHIERS/OUTILS/BASES DE DONNEES/RHOMEO/2012/CEN-38_RHOMEO_BDD_2012'
|
||||
ACCESS_FILE = 'BD_Rhomeo_data.mdb'
|
||||
DICT_COLS = {
|
||||
'obse_id':'id_origine',
|
||||
}
|
||||
|
||||
|
||||
def get_columns_oftable(tab,sch,con):
|
||||
res = con.dialect.get_columns(con,tab,sch)
|
||||
return [x['name'] for x in res]
|
||||
|
||||
|
||||
def get_columns_dtypes(tab,sch,con,df:pd.DataFrame=None):
|
||||
res = con.dialect.get_columns(con,tab,sch)
|
||||
if isinstance(df,pd.DataFrame):
|
||||
return {x['name']:x['type'] for x in res if x['name'] in df.columns}
|
||||
else:
|
||||
return {x['name']:x['type'] for x in res}
|
||||
|
||||
|
||||
def match_columns(df,tab,sch,con):
|
||||
cols_saisie = get_columns_oftable(tab,sch,con)
|
||||
obs_cols = df.columns
|
||||
return obs_cols[obs_cols.isin(cols_saisie)]
|
||||
|
||||
|
||||
def lower(df):
|
||||
for c in df.columns :
|
||||
df[c] = df[c].str.lower() if df[c].apply(type).eq(str).all() else df[c]
|
||||
return df
|
||||
|
||||
|
||||
def get_odat_obse(id_obse:gpd.np.ndarray|list|pd.Series|int=None):
|
||||
sql = '''
|
||||
SELECT
|
||||
o.odat_obse_id,
|
||||
c.choi_nom,
|
||||
l.list_nom,
|
||||
o.odat_nom
|
||||
FROM {sch}.rnf_odat o
|
||||
LEFT JOIN {sch}.rnf_choi c ON c.choi_id = o.odat_choi_id
|
||||
LEFT JOIN {sch}.rnf_list l ON l.list_id = o.odat_c_list_id
|
||||
'''.format(sch=postgis_sch)
|
||||
|
||||
if isinstance(id_obse,int) :
|
||||
sql += ' WHERE odat_obse_id = %i'%id_obse
|
||||
elif isinstance(id_obse,(list,gpd.pd.Series,gpd.np.ndarray)) :
|
||||
sql += ' WHERE odat_obse_id IN {lst_id}'.format(lst_id=tuple(id_obse))
|
||||
|
||||
result = pd.read_sql_query(sql,con_sicen,index_col=['odat_obse_id','list_nom']).replace({r' \(RhoMeO\)':'',r'\?':'',r'[.]':''},regex=True)
|
||||
|
||||
df = lower(result).pivot(columns='choi_nom',values='odat_nom').reset_index(drop=False)
|
||||
|
||||
return format_comportement(df)
|
||||
|
||||
|
||||
def format_comportement(oda:pd.DataFrame):
|
||||
DICT = {
|
||||
'exuvie/émergent':'ODO_Exuvie/émergence',
|
||||
'mâles+femelles':'ODO_Mâles+Femelles',
|
||||
'autre':'ODO_Autre',
|
||||
'territoriale':'ODO_Territorial',
|
||||
'ponte':'ODO_Ponte',
|
||||
'accouplement':'ODO_Tandem',
|
||||
'tandem':'ODO_Tandem',
|
||||
'aucun':None,
|
||||
}
|
||||
return oda.replace(DICT)
|
||||
|
||||
|
||||
def format_odat(df,odat):
|
||||
DICT_COLS = {
|
||||
'comportement':'reprostatut',
|
||||
'strate':'type_effectif',
|
||||
'rmq_odat':'remarque_obs'
|
||||
}
|
||||
|
||||
|
||||
df_odat = df[['obse_id','obse_nombre','obse_nom','cd_nom','ordre']].merge(
|
||||
odat,
|
||||
left_on='obse_id',right_on='odat_obse_id'
|
||||
)
|
||||
|
||||
nb_cols = odat.columns[odat.columns.str.startswith('nb')]
|
||||
df_odat['rmq_odat'] = None
|
||||
for c in nb_cols:
|
||||
test1 = df_odat.obse_nombre!=df_odat[c]
|
||||
test2 = df_odat[c].notna()
|
||||
test3 = df_odat.rmq_odat.notna()
|
||||
test = test1&test2&test3
|
||||
tes = test1&test2&~test3
|
||||
if any(tes) or any(test):
|
||||
tt = '{rmq_o}; {c}:{data}'.format(rmq_o=df_odat[test].rmq_odat,c=c,data=df_odat[test][c])
|
||||
df_odat.loc[test,'rmq_odat'] = df_odat[test].rmq_odat+';'+c+':'+df_odat[test][c]
|
||||
df_odat.loc[tes,'rmq_odat'] = c+':'+df_odat[tes][c]
|
||||
|
||||
return df_odat.rename(columns=DICT_COLS)[['odat_obse_id',*DICT_COLS.values()]]
|
||||
|
||||
|
||||
|
||||
def get_obs_serena(insicen:bool=None):
|
||||
sql = '''
|
||||
SELECT o.*,
|
||||
s.site_nom,
|
||||
s.site_ref_sig,
|
||||
CASE WHEN t.taxo_id = 203471 THEN (SELECT cd_nom FROM inpn.taxref WHERE cd_nom = '54213' )
|
||||
WHEN t.taxo_id = 203491 THEN (SELECT cd_nom FROM inpn.taxref WHERE cd_nom = '521494' )
|
||||
WHEN tax.cd_nom IS NULL THEN ta.cd_nom::text
|
||||
ELSE tax.cd_nom
|
||||
END cd_nom,
|
||||
CASE WHEN t.taxo_id = 203471 THEN (SELECT phylum FROM inpn.taxref WHERE cd_nom = '54213' )
|
||||
WHEN t.taxo_id = 203491 THEN (SELECT phylum FROM inpn.taxref WHERE cd_nom = '521494' )
|
||||
WHEN tax.phylum IS NULL THEN ta.phylum
|
||||
ELSE tax.phylum
|
||||
END phylum,
|
||||
CASE WHEN t.taxo_id = 203471 THEN (SELECT classe FROM inpn.taxref WHERE cd_nom = '54213' )
|
||||
WHEN t.taxo_id = 203491 THEN (SELECT classe FROM inpn.taxref WHERE cd_nom = '521494' )
|
||||
WHEN tax.classe IS NULL THEN ta.classe
|
||||
ELSE tax.classe
|
||||
END classe,
|
||||
CASE WHEN t.taxo_id = 203471 THEN (SELECT ordre FROM inpn.taxref WHERE cd_nom = '54213' )
|
||||
WHEN t.taxo_id = 203491 THEN (SELECT ordre FROM inpn.taxref WHERE cd_nom = '521494' )
|
||||
WHEN tax.ordre IS NULL THEN ta.ordre
|
||||
ELSE tax.ordre
|
||||
END ordre,
|
||||
CASE WHEN t.taxo_id = 203471 THEN (SELECT famille FROM inpn.taxref WHERE cd_nom = '54213' )
|
||||
WHEN t.taxo_id = 203491 THEN (SELECT famille FROM inpn.taxref WHERE cd_nom = '521494' )
|
||||
WHEN tax.famille IS NULL THEN ta.famille
|
||||
ELSE tax.famille
|
||||
END famille,
|
||||
CASE WHEN t.taxo_id = 203471 THEN (SELECT nom_complet FROM inpn.taxref WHERE cd_nom = '54213' )
|
||||
WHEN t.taxo_id = 203491 THEN (SELECT nom_complet FROM inpn.taxref WHERE cd_nom = '521494' )
|
||||
WHEN tax.nom_complet IS NULL THEN ta.nom_complet
|
||||
ELSE tax.nom_complet
|
||||
END nom_complet,
|
||||
CASE WHEN t.taxo_id = 203471 THEN (SELECT nom_vern FROM inpn.taxref WHERE cd_nom = '54213' )
|
||||
WHEN t.taxo_id = 203491 THEN (SELECT nom_vern FROM inpn.taxref WHERE cd_nom = '521494' )
|
||||
WHEN tax.nom_vern IS NULL THEN ta.nom_vern
|
||||
ELSE tax.nom_vern
|
||||
END nom_vern,
|
||||
CASE WHEN tmpo.ogll_lon = '999' THEN tmps.sgll_lon::float
|
||||
ELSE tmpo.ogll_lon::float
|
||||
END lon,
|
||||
CASE WHEN tmpo.ogll_lat = '999' THEN tmps.sgll_lat::float
|
||||
ELSE tmpo.ogll_lat::float
|
||||
END lat
|
||||
FROM {sch}.rnf_obse o
|
||||
LEFT JOIN {sch}.rnf_relv r ON r.relv_id = o.obse_relv_id
|
||||
LEFT JOIN {sch}.rnf_site s ON s.site_id = o.obse_site_id
|
||||
LEFT JOIN {sch}.tmp_sgll tmps ON tmps.sgll_site_id = o.obse_site_id
|
||||
LEFT JOIN {sch}.tmp_ogll tmpo ON tmpo.ogll_obse_id = o.obse_id
|
||||
LEFT JOIN serenarefe.rnf_taxo t ON t.taxo_id = o.obse_taxo_id
|
||||
LEFT JOIN inpn.taxons_isere_absents_taxref ta ON ta.id_taxon = o.obse_taxo_id
|
||||
LEFT JOIN inpn.taxref tax ON t.taxo_mnhn_id = tax.cd_nom::int
|
||||
--LEFT JOIN {sch}.rnf_odat od ON od.odat_obse_id = o.obse_id
|
||||
|
||||
'''.format(sch=postgis_sch)
|
||||
|
||||
if insicen is not None :
|
||||
sql_sicen = 'SELECT DISTINCT id_origine FROM saisie.saisie_observation WHERE id_lot=3'
|
||||
sicen_obs = pd.read_sql_query(sql_sicen,con_sicen)
|
||||
if insicen is True :
|
||||
sql += 'WHERE obse_id IN {lst_id}'.format(lst_id=tuple(sicen_obs.id_origine.astype(int)))
|
||||
if insicen is False :
|
||||
sql += 'WHERE obse_id NOT IN {lst_id}'.format(lst_id=tuple(sicen_obs.id_origine.astype(int)))
|
||||
|
||||
return pd.read_sql_query(sql,con_sicen)\
|
||||
.dropna(axis=1,how='all')
|
||||
|
||||
|
||||
def list_releve(id_relv=None,source='postgis'):
|
||||
if source=='postgis':
|
||||
|
||||
df_choi = pd.read_sql_table('rnf_choi',con_sicen,postgis_sch)\
|
||||
.dropna(axis=1,how='all')
|
||||
df_relv = pd.read_sql_table('rnf_relv',con_sicen,postgis_sch)\
|
||||
.dropna(axis=1,how='all')
|
||||
|
||||
# Correction coquille
|
||||
df_relv.loc[df_relv.relv_categ_choi_id==100800,'relv_categ_choi_id'] = 100008
|
||||
|
||||
# Jointure relevé / catégorie
|
||||
df = df_relv.merge(df_choi[['choi_id','choi_nom']],how='inner',left_on='relv_categ_choi_id',right_on='choi_id')\
|
||||
.drop(columns=['relv_categ_choi_id'])
|
||||
|
||||
# Mise en forme des dates
|
||||
lstcol_dat = df.columns[df.columns.str.contains('date')]
|
||||
for lcd in lstcol_dat:
|
||||
# df[lcd] = df[lcd].replace({'VIDE':None})
|
||||
df.loc[df[lcd]=='2000 à 2001',lcd] = '2000' if '1date' in lcd else '2001'
|
||||
df[lcd].replace({'VIDE':None},inplace=True)
|
||||
df[lcd] = pd.to_datetime(df[lcd])
|
||||
|
||||
if id_relv is not None:
|
||||
if isinstance(id_relv,int) : filtre = df.relv_id == id_relv
|
||||
elif isinstance(id_relv,(list,gpd.pd.Series,gpd.np.ndarray)) : filtre = df.relv_id.isin([*id_relv])
|
||||
return df[filtre]
|
||||
else:
|
||||
return df
|
||||
|
||||
|
||||
def get_serena_obs(srce_id=None,source:str='access'):
|
||||
DICT_STR = {
|
||||
'avenir':'cen isère',
|
||||
'be':'personnel'
|
||||
}
|
||||
if source == 'postgis':
|
||||
sql = 'SELECT user_id,user_srce_id FROM {sch}.rnf_user'.format(sch=postgis_sch)
|
||||
if srce_id is not None:
|
||||
if isinstance(srce_id,int) :
|
||||
sql += ' WHERE user_srce_id = %i'%srce_id
|
||||
elif isinstance(srce_id,(list,gpd.pd.Series,gpd.np.ndarray)) :
|
||||
sql += ' WHERE user_srce_id IN {lst_id}'.format(lst_id=tuple(srce_id))
|
||||
return lower(pd.read_sql_query(sql,con_sicen))
|
||||
|
||||
elif source == 'access':
|
||||
df = mdb.read_table(path.join(ACCESS_PATH,ACCESS_FILE),'rev_obs')
|
||||
if srce_id is not None:
|
||||
if isinstance(srce_id,int) :
|
||||
df = df[df.rev_observateur==srce_id]
|
||||
elif isinstance(srce_id,(list,gpd.pd.Series,gpd.np.ndarray)) :
|
||||
df = df[df.rev_observateur.isin([*srce_id])]
|
||||
df[['nom','prenom']] = df.nom_observateur.str.split(' ',1,expand=True)
|
||||
|
||||
return lower(df).replace(DICT_STR)
|
||||
|
||||
|
||||
def get_sicen_obs(personne_id=None):
|
||||
sql = 'SELECT id_personne,prenom,nom,id_structure,nom_structure FROM md.personne JOIN md.structure USING (id_structure)'.format(sch=postgis_sch)
|
||||
if personne_id is not None:
|
||||
if isinstance(personne_id,int) :
|
||||
sql += ' WHERE personne_id = %i'%personne_id
|
||||
elif isinstance(personne_id,str) :
|
||||
sql += ' WHERE personne_id = %i'%int(personne_id)
|
||||
elif isinstance(personne_id,(list,gpd.pd.Series,gpd.np.ndarray)) :
|
||||
sql += ' WHERE personne_id IN {lst_id}'.format(lst_id=tuple(personne_id))
|
||||
return lower(pd.read_sql_query(sql,con_sicen))
|
||||
|
||||
def get_sicen_etude():
|
||||
sql = 'SELECT * FROM md.etude'
|
||||
return pd.read_sql_query(sql, con_sicen)
|
||||
|
||||
|
||||
def get_serena_choi(choi_id:gpd.np.ndarray|list|pd.Series|int=None):
|
||||
''''''
|
||||
sql = 'SELECT choi_id,choi_nom FROM {sch}.rnf_choi'.format(sch=postgis_sch)
|
||||
|
||||
if isinstance(choi_id,int) :
|
||||
sql += ' WHERE choi_id = %i'%choi_id
|
||||
elif isinstance(choi_id,(list,gpd.pd.Series,gpd.np.ndarray)) :
|
||||
sql += ' WHERE choi_id IN {lst_id}'.format(lst_id=tuple(choi_id))
|
||||
|
||||
return lower(pd.read_sql_query(sql,con_sicen))
|
||||
|
||||
|
||||
def get_serena_site(site_id:gpd.np.ndarray|list|pd.Series|int=None):
|
||||
sql = 'SELECT site_id,site_nom FROM {sch}.rnf_site'.format(sch=postgis_sch)
|
||||
|
||||
if isinstance(site_id,int) :
|
||||
sql += ' WHERE site_id = %i'%site_id
|
||||
elif isinstance(site_id,(list,gpd.pd.Series,gpd.np.ndarray)) :
|
||||
sql += ' WHERE site_id IN {lst_id}'.format(lst_id=tuple(site_id))
|
||||
return pd.read_sql_query(sql,con_sicen)
|
||||
|
||||
|
||||
def get_sicen_pcol():
|
||||
sql = '''SELECT * FROM "md"."protocole";'''
|
||||
return lower(pd.read_sql_query(sql,con_sicen))
|
||||
|
||||
|
||||
def _crsp(left:pd.DataFrame,right:pd.DataFrame,lefton,righton,idleft,idright):
|
||||
df = left.merge(right,left_on=lefton,right_on=righton)
|
||||
return dict(zip(df[idleft],df[idright]))
|
||||
|
||||
|
||||
def crsp_relv(relv:pd.Series):
|
||||
seren_relv = lower(list_releve(relv.unique()))
|
||||
sicen_relv = lower(get_sicen_etude())
|
||||
DICT_RELV_ID = _crsp(left=seren_relv,right=sicen_relv,lefton='relv_nom',righton='nom_etude',idleft='relv_id',idright='id_etude')
|
||||
return relv.replace(DICT_RELV_ID)
|
||||
|
||||
|
||||
def crsp_obs(obs:pd.Series):
|
||||
obs_seren = get_serena_obs(obs.unique())
|
||||
obs_sicen = get_sicen_obs()
|
||||
DICT_OBS_ID = _crsp(left=obs_seren,right=obs_sicen,lefton=['nom','structure_obs'],righton=['nom','nom_structure'],idleft='rev_observateur',idright='id_personne')
|
||||
return obs.replace(DICT_OBS_ID)
|
||||
|
||||
|
||||
def crsp_pcol(pcol:pd.Series):
|
||||
pcol_seren = get_serena_choi(pcol.unique())
|
||||
pcol_sicen = get_sicen_pcol()
|
||||
DICT_PCOL_ID = _crsp(left=pcol_seren,right=pcol_sicen,lefton='choi_nom',righton='libelle',idleft='choi_id',idright='id_protocole')
|
||||
return pcol.replace(DICT_PCOL_ID)
|
||||
|
||||
|
||||
def crsp_valid(valid:pd.Series):
|
||||
pcol_seren = get_serena_choi(valid.unique())
|
||||
pcol_sicen = get_sicen_pcol()
|
||||
DICT_PCOL_ID = _crsp(left=pcol_seren,right=pcol_sicen,lefton='choi_nom',righton='libelle',idleft='choi_id',idright='id_protocole')
|
||||
return valid.replace(DICT_PCOL_ID)
|
||||
|
||||
|
||||
def crsp_abond(abond:pd.Series):
|
||||
ab_seren = get_serena_choi(abond.dropna().unique())
|
||||
DICT_ABOND_ID = dict(zip(ab_seren.choi_id,ab_seren.choi_nom))
|
||||
return abond.replace(DICT_ABOND_ID)
|
||||
|
||||
|
||||
def crsp_site(site:pd.Series):
|
||||
sit_seren = get_serena_site(site.dropna().unique())
|
||||
DICT_SITE_ID = dict(zip(sit_seren.site_id,sit_seren.site_nom))
|
||||
return site.replace(DICT_SITE_ID)
|
||||
|
||||
|
||||
def get_structure_id(id_pers:gpd.np.ndarray|list|pd.Series|int=None):
|
||||
sql = 'SELECT id_personne, id_structure FROM md.personne'
|
||||
if isinstance(id_pers,int) :
|
||||
sql += ' WHERE id_personne = %i'%id_pers
|
||||
elif isinstance(id_pers,(list,gpd.pd.Series,gpd.np.ndarray)) :
|
||||
sql += ' WHERE id_personne IN {lst_id}'.format(lst_id=tuple(id_pers))
|
||||
return pd.read_sql_query(sql,con_sicen)
|
||||
|
||||
|
||||
def format_effectif(DF):
|
||||
df = DF.copy()
|
||||
isnum = df.effectif_min.str.isnumeric()
|
||||
df.loc[isnum,'effectif'] = df[isnum].effectif_min.copy()
|
||||
df.loc[isnum,'effectif_min'] = None
|
||||
df.loc[~isnum,'effectif_textuel'] = df[~isnum].effectif_min.copy()
|
||||
df.loc[~isnum,'effectif_min'] = df[~isnum].effectif_textuel.str.split(r'[-+]',expand=True)[0]
|
||||
df.loc[~isnum,'effectif_max'] = df[~isnum].effectif_textuel.str.split(r'[-+]',expand=True)[1]
|
||||
return df.replace({'':None})
|
||||
|
||||
def to_sicen(DF):
|
||||
|
||||
CRSP_COLUMNS = {
|
||||
'obse_id':'id_origine',
|
||||
'obse_relv_id':'id_etude',
|
||||
'obse_obsv_id':'observateur',
|
||||
'obse_detm_id':'validateur',
|
||||
'obse_date':'date_obs',
|
||||
# 'obse_site_id':'localisation',
|
||||
'obse_pcole_choi_id':'id_protocole',
|
||||
'obse_validat_choi_id':'statut_validation',
|
||||
'obse_confid_choi_id':'diffusable',
|
||||
'obse_abond_choi_id':'effectif_textuel',
|
||||
'obse_nombre':'effectif_min',
|
||||
'site_nom':'localisation',
|
||||
'site_ref_sig':'id_waypoint',
|
||||
'lat':'latitude',
|
||||
'lon':'longitude',
|
||||
}
|
||||
|
||||
df = DF.copy()
|
||||
if df.geometry.name != 'geometrie' :
|
||||
df.rename_geometry('geometrie',inplace=True)
|
||||
|
||||
df.obse_relv_id = crsp_relv(df.obse_relv_id)
|
||||
df.obse_obsv_id = crsp_obs(df.obse_obsv_id)
|
||||
df.obse_detm_id = crsp_obs(df.obse_detm_id).astype(int)
|
||||
df.obse_date = pd.to_datetime(df.obse_date)
|
||||
# df.obse_site_id = crsp_site(df.obse_site_id)
|
||||
df.obse_pcole_choi_id = crsp_pcol(df.obse_pcole_choi_id)
|
||||
df.loc[df.obse_validat_choi_id==100373,'obse_validat_choi_id'] = 'validée'
|
||||
df.loc[df.obse_confid_choi_id==100473,'obse_confid_choi_id'] = True
|
||||
df.obse_abond_choi_id = crsp_abond(df.obse_abond_choi_id)
|
||||
df.rename(columns=CRSP_COLUMNS,inplace=True)
|
||||
|
||||
# Tag du lot SERENA
|
||||
df['id_lot'] = 3
|
||||
|
||||
# Jointure des id_structure
|
||||
struct = get_structure_id(df.observateur.unique())
|
||||
df = df.merge(struct,how='left',left_on='observateur', right_on='id_personne')\
|
||||
.rename(columns={'id_structure':'structure'})
|
||||
cols = match_columns(df,'saisie_observation','saisie',con_sicen)
|
||||
|
||||
|
||||
# Formatage des effectifs
|
||||
return format_effectif(df[cols])
|
||||
|
||||
|
||||
def drop_cdnom_missing(df):
|
||||
isna = df.cd_nom.isna()
|
||||
idx_na = df[isna].index
|
||||
return df.drop(idx_na)
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
obs_serena = get_obs_serena(insicen=True)
|
||||
isna = obs_serena.cd_nom.isna()
|
||||
|
||||
# Recréation des géométries
|
||||
gdf_obs = obs_serena.set_geometry(gpd.points_from_xy(obs_serena.lon,obs_serena.lat))
|
||||
gdf_obs.set_crs(4326,inplace=True)
|
||||
gdf_obs.to_crs(2154,inplace=True)
|
||||
|
||||
# Identification des données RhoMeo
|
||||
is_rhomeo = gdf_obs.obse_nom.str.contains('rhoméo',na=False,case=False)
|
||||
gdf_obs.loc[is_rhomeo,['obse_id','obse_nom','obse_date','cd_nom']]
|
||||
obs_rhomeo = gdf_obs[is_rhomeo]\
|
||||
.dropna(axis=1,how='all')\
|
||||
.drop(columns=['obse_habi_id'])
|
||||
rhoisna = obs_rhomeo.cd_nom.isna()
|
||||
obs_rhomeo = drop_cdnom_missing(obs_rhomeo)
|
||||
|
||||
odat = get_odat_obse(obs_rhomeo.obse_id)
|
||||
|
||||
OBS_RHOMEO = obs_rhomeo.merge(
|
||||
format_odat(obs_rhomeo,odat),
|
||||
left_on='obse_id',right_on='odat_obse_id',how='left'
|
||||
).drop(columns='odat_obse_id')
|
||||
|
||||
OBS_RHOMEO.to_file('/home/colas/Documents/9_PROJETS/4_SICEN/RECOVERY/rhomeo_data_notin_sicen.gpkg')
|
||||
|
||||
RES_OBS = to_sicen(OBS_RHOMEO)
|
||||
RES_OBS.to_postgis(
|
||||
'saisie_observation',
|
||||
con_sicen,
|
||||
'saisie',
|
||||
if_exists='append',
|
||||
index=False,
|
||||
dtype=get_columns_dtypes(
|
||||
'saisie_observation',
|
||||
'saisie',
|
||||
con_sicen,
|
||||
RES_OBS
|
||||
)
|
||||
)
|
||||
# .to_file('/home/colas/Documents/9_PROJETS/4_SICEN/RECOVERY/rhomeo_data_notin_sicen.gpkg')
|
||||
|
||||
|
||||
update_to_sql(
|
||||
RES_OBS[['id_lot','id_origine','effectif','effectif_min']],
|
||||
con_sicen,
|
||||
'saisie_observation',
|
||||
'saisie',
|
||||
['id_lot','id_origine'],
|
||||
dtype=get_columns_dtypes(
|
||||
'saisie_observation',
|
||||
'saisie',
|
||||
con_sicen,
|
||||
RES_OBS[['id_lot','id_origine','effectif','effectif_min']]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# Récupération des relevés
|
||||
id_relvrho = gdf_obs[is_rhomeo].obse_relv_id.unique()
|
||||
relv = list_releve(id_relv=id_relvrho)
|
||||
relv_rh = relv.relv_nom
|
||||
Reference in New Issue
Block a user