init depot renew
This commit is contained in:
@@ -0,0 +1,322 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
|
||||
from pycen import con_sicen
|
||||
import geopandas as gpd
|
||||
|
||||
DICT_TAXREF_OLDNAME = {
|
||||
116744 : 521658, # Quercus petraea
|
||||
# 9999056 : 0, # Micromammalia sp.
|
||||
9999077 : 183820, # Arachnida sp.
|
||||
9999078 : 187496, # Fungi sp.
|
||||
# 9999081 : 0, # Heterocera sp.
|
||||
9999058 : 195005, # Myotis sp.
|
||||
}
|
||||
|
||||
DICT_TAXREF = {
|
||||
# 116744:{'cd_nom':521658,'lb_nom':'Quercus petraea (Matt.) Liebl., 1784'}, # Quercus petraea
|
||||
# 9999077:{'cd_nom':183820,'lb_nom':'Arachnida Cuvier, 1812'}, # Arachnida sp.
|
||||
|
||||
# Espèce domestique
|
||||
9999031 : {'cd_nom':441709,'lb_nom':'Cairina moschata (Linnaeus, 1758)'}, # Cairina moschata f. domestica
|
||||
|
||||
# GROUPE
|
||||
9999005: {'cd_nom':194069,'lb_nom':'Leucanthemum Mill., 1754'}, # Leucanthemum vulgare (#groupe) (#ref)
|
||||
9999014: {'cd_nom':197281,'lb_nom':'Rubus L., 1753 [nom. et typ. cons.]'}, # Rubus fruticosus (#groupe) (#ref)
|
||||
9999017: {'cd_nom':198226,'lb_nom':'Taraxacum F.H.Wigg., 1780'}, # Taraxacum officinale (#groupe) (#ref)
|
||||
9999019: {'cd_nom':198449,'lb_nom':'Thymus L., 1753'}, # Thymus serpyllum (#groupe) (#ref)
|
||||
9999020: {'cd_nom':198902,'lb_nom':'Vicia L., 1753'}, # Vicia sativa (#groupe) (#ref)
|
||||
|
||||
# COMPLEXE
|
||||
9999075: {'cd_nom':196296,'lb_nom':'Pipistrellus Kaup, 1829'}, # Pipistrellus nathusii / kuhlii (#complexe) (#ref)
|
||||
9999074: {'cd_nom':197040,'lb_nom':'Rana Linnaeus, 1758'}, # Rana dalmatina / temporaria (#complexe) (#ref)
|
||||
9999057: {'cd_nom':195005,'lb_nom':'Myotis Kaup, 1829'}, # Myotis myotis / blythii (#complexe) (#ref)
|
||||
9999050: {'cd_nom':193993,'lb_nom':'Leptidea Billberg, 1820'}, # Leptidea sinapis / reali (#complexe) (#ref)
|
||||
9999066: {'cd_nom':196980,'lb_nom':'Pyrgus Hübner, 1819'}, # Pyrgus malvae / malvoides (#complexe) (#ref)
|
||||
9999063: {'cd_nom':444436,'lb_nom':'Pelophylax Fitzinger, 1843'}, # Pelophylax kl. esculentus / lessonae (#complex...
|
||||
9999054: {'cd_nom':194481,'lb_nom':'Martes Pinel, 1792'}, # Martes martes / foina (#complexe) (#ref)
|
||||
9999037: {'cd_nom':191029,'lb_nom':'Colias Fabricius, 1807'}, # Colias alfacariensis / hyale (#complexe) (#ref)
|
||||
9999023: {'cd_nom':188772,'lb_nom':'Acrocephalus J. A. Naumann & J. F. Naumann, 1811'}, # Acrocephalus palustris / scirpaceus (#complexe...
|
||||
9999064: {'cd_nom':4280,'lb_nom':'Phylloscopus collybita (Vieillot, 1817)'}, # Phylloscopus collybita tristis / "abietinus" ...
|
||||
9999046: {'cd_nom':192539,'lb_nom':'Felis Linnaeus, 1758'}, # Felis silvestris / catus (#complexe) (#ref)
|
||||
9999080: {'cd_nom':194357,'lb_nom':'Lysandra Hemming, 1933'},# Lysandra coridon / hispana (#complexe)
|
||||
9999033: {'cd_nom':886228,'lb_nom':'Acanthis Borkhausen, 1797'}, # Carduelis flammea flammea / cabaret / Carduelis hornemanni (#complexe)
|
||||
9999041: {'cd_nom':4503,'lb_nom':'Corvus corone Linnaeus, 1758'}, # Corvus corone corone / cornix (#complexe)
|
||||
9999042: {'cd_nom':186239,'lb_nom':'Vespertilionidae J. E. Gray, 1821'}, # Eptesicus / Nyctalus sp. (#complexe) (#ref)
|
||||
9999082: {'cd_nom':195005,'lb_nom':'Myotis Kaup, 1829'}, # Myotis daubentonii/Myotis mystacinus (#complexe)
|
||||
9999083: {'cd_nom':699094,'lb_nom':'Yangochiroptera Koopman, 1985'}, # Pipistrellus/Miniopterus (#complexe)
|
||||
}
|
||||
|
||||
def get_pheno_genre(esp_fx, df):
|
||||
nom = esp_fx
|
||||
return
|
||||
|
||||
|
||||
|
||||
def _vm_synthese_observations_(where=''):
|
||||
# Lecture des données
|
||||
sql = 'SELECT * FROM saisie.vm_synthese_observations '
|
||||
df = gpd.read_postgis(sql+where,con_sicen)
|
||||
|
||||
# drop Micromammalia sp.
|
||||
drp = df.loc[df.cd_nom == 9999056].index
|
||||
df.drop(drp,inplace=True)
|
||||
|
||||
is_group = df.nom_complet.str.contains('#group')
|
||||
is_complex = df.nom_complet.str.contains('#complex')
|
||||
is_domestic = df.nom_complet.str.contains('domestic')
|
||||
lst_group = [*df[is_group].nom_complet.unique()]
|
||||
lst_complex = [*df[is_complex].nom_complet.unique()]
|
||||
lst_domestic = [*df[is_domestic].nom_complet.unique()]
|
||||
|
||||
# df['group'] = None
|
||||
# df['complex'] = None
|
||||
# df.loc[is_group,'group'] = df[is_group].nom_complet.copy()
|
||||
# df.loc[is_complex,'complex'] = df[is_complex].nom_complet.copy()
|
||||
df['rmq_taxonomie'] = None
|
||||
df.loc[is_complex|is_group|is_domestic,'rmq_taxonomie'] = df[is_complex|is_group|is_domestic].nom_complet.copy()
|
||||
|
||||
for cd in df[is_complex|is_group|is_domestic].cd_nom.unique():
|
||||
lb = df[df.cd_nom==cd].nom_complet.unique()[0]
|
||||
cdn = {cd:DICT_TAXREF[cd]['cd_nom']}
|
||||
lbn = {lb:DICT_TAXREF[cd]['lb_nom']}
|
||||
|
||||
df.nom_complet.replace(lbn,inplace=True)
|
||||
df.nom_latin.replace(lbn,inplace=True)
|
||||
df.cd_nom.replace(cdn,inplace=True)
|
||||
|
||||
# Transform geom TO WKT format
|
||||
df['geom_wkt'] = df.geometry.to_wkt()
|
||||
# Récup code insee département
|
||||
df['insee_dep'] = df.insee_commune.str[:2]
|
||||
# Mise en forme de la précision de la données
|
||||
df.precision = format_precision(df.precision)
|
||||
# Actualisation des cd_nom
|
||||
df.cd_nom.replace(DICT_TAXREF_OLDNAME,inplace=True)
|
||||
|
||||
# Mise en forme des effectifs
|
||||
df.effectif = format_effectif(df.effectif.copy())
|
||||
df.effectif_min = format_effectif(df.effectif_min.copy())
|
||||
df.effectif_max = format_effectif(df.effectif_max.copy())
|
||||
df.loc[df.effectif.isna(),'effectif'] = df[df.effectif.isna()].effectif_min
|
||||
df.effectif.fillna('1',inplace=True)
|
||||
df.loc[df.effectif_max.isna(),'effectif_max'] = df[df.effectif_max.isna()].effectif
|
||||
df.effectif = df.effectif.astype(int)
|
||||
df.effectif_max = df.effectif_max.astype(int)
|
||||
|
||||
check_effectif = df['effectif'].astype(float) > df['effectif_min'].astype(float)
|
||||
if not df[check_effectif].empty:
|
||||
print('WARNING : "effectif" > "effectif_min"')
|
||||
print(' nrows : %s'%df[check_effectif].shape[0])
|
||||
|
||||
df.loc[check_effectif,'effectif_max'] = df[check_effectif].effectif.copy()
|
||||
df.loc[check_effectif,'effectif'] = df[check_effectif].effectif_min.copy()
|
||||
print( 'effectif TO effectif_max : OK !')
|
||||
|
||||
df.effectif_min = df.effectif.copy()
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def format_precision(lst):
|
||||
return lst.copy().replace({
|
||||
'GPS': '5',
|
||||
'0 à 10m': '5',
|
||||
'10 à 100m': '50',
|
||||
'100 à 500m': '250',
|
||||
'500 à 1000m': '750',
|
||||
'> 1000m': '2000',
|
||||
'lieu-dit': '2500',
|
||||
'commune': '5000',
|
||||
})
|
||||
|
||||
def format_effectif(lst):
|
||||
lst.loc[lst.notna()] = lst[lst.notna()].astype(int).astype(str)
|
||||
return lst
|
||||
|
||||
|
||||
def additional_data(df,columns):
|
||||
df['additional_data'] = df[columns].to_json(orient="records",force_ascii=False)
|
||||
df.drop(columns=columns,inplace=True)
|
||||
return df
|
||||
|
||||
|
||||
class flore:
|
||||
|
||||
def all():
|
||||
w = "WHERE regne = 'Plantae'"
|
||||
res = _vm_synthese_observations_(w)
|
||||
# return additional_data(res,['strate_flore','effectif_textuel'])
|
||||
return res
|
||||
|
||||
def vasculaire():
|
||||
w = "WHERE group1_inpn = 'Plantes vasculaires'"
|
||||
res = _vm_synthese_observations_(w)
|
||||
# return additional_data(res,['strate_flore','effectif_textuel'])
|
||||
return res
|
||||
|
||||
def bryophyte():
|
||||
w = "WHERE group1_inpn = 'Bryophytes'"
|
||||
res = _vm_synthese_observations_(w)
|
||||
# return additional_data(res,['strate_flore','effectif_textuel'])
|
||||
return res
|
||||
|
||||
def characee():
|
||||
w = "WHERE famille = 'Characeae'"
|
||||
res = _vm_synthese_observations_(w)
|
||||
# return additional_data(res,['strate_flore','effectif_textuel'])
|
||||
return res
|
||||
|
||||
|
||||
class faune:
|
||||
def all():
|
||||
w = "WHERE regne = 'Animalia'"
|
||||
return _vm_synthese_observations_(w)
|
||||
|
||||
class invertebre:
|
||||
def all():
|
||||
w = "WHERE regne = 'Animalia' AND phylum <> 'Chordata'"
|
||||
return _vm_synthese_observations_(w)
|
||||
|
||||
def odonate():
|
||||
w = "WHERE ordre = 'Odonata'"
|
||||
return _vm_synthese_observations_(w)
|
||||
def orthoptere():
|
||||
w = "WHERE ordre = 'Orthoptera'"
|
||||
return _vm_synthese_observations_(w)
|
||||
def lepidoptere():
|
||||
w = "WHERE ordre = 'Lepidoptera'"
|
||||
return _vm_synthese_observations_(w)
|
||||
def mollusque():
|
||||
w = "WHERE group1_inpn = 'Mollusques'"
|
||||
return _vm_synthese_observations_(w)
|
||||
def crustacees():
|
||||
w = "WHERE group2_inpn = 'Crustacés'"
|
||||
return _vm_synthese_observations_(w)
|
||||
def ascomycete():
|
||||
w = "WHERE ordre = 'Ascomycètes'"
|
||||
return _vm_synthese_observations_(w)
|
||||
def autre():
|
||||
w = """
|
||||
WHERE regne = 'Animalia'
|
||||
AND phylum <> 'Chordata'
|
||||
AND ordre <> 'Odonata'
|
||||
AND ordre <> 'Orthoptera'
|
||||
AND ordre <> 'Lepidoptera'
|
||||
AND group1_inpn <> 'Mollusques'
|
||||
AND group2_inpn <> 'Crustacés'
|
||||
AND ordre <> 'Ascomycètes'
|
||||
"""
|
||||
return _vm_synthese_observations_(w)
|
||||
|
||||
class vertebre:
|
||||
def all():
|
||||
w = "WHERE regne = 'Animalia' AND phylum = 'Chordata'"
|
||||
return _vm_synthese_observations_(w)
|
||||
|
||||
def mammifere():
|
||||
w = "WHERE group2_inpn = 'Mammifères' AND ordre <> 'Chiroptera'"
|
||||
return _vm_synthese_observations_(w)
|
||||
def chiroptere():
|
||||
w = "WHERE ordre = 'Chiroptera'"
|
||||
return _vm_synthese_observations_(w)
|
||||
def oiseau():
|
||||
w = "WHERE group2_inpn = 'Oiseaux'"
|
||||
return _vm_synthese_observations_(w)
|
||||
def reptile():
|
||||
w = "WHERE group2_inpn = 'Reptiles'"
|
||||
return _vm_synthese_observations_(w)
|
||||
def amphibien():
|
||||
w = "WHERE group2_inpn = 'Amphibiens'"
|
||||
return _vm_synthese_observations_(w)
|
||||
def poisson():
|
||||
w = "WHERE group2_inpn = 'Poissons'"
|
||||
return _vm_synthese_observations_(w)
|
||||
|
||||
|
||||
def champignon():
|
||||
w = "WHERE regne IN ('Fungi','Chromista')"
|
||||
return _vm_synthese_observations_(w)
|
||||
|
||||
|
||||
def export_csv(df,path_name):
|
||||
df.dropna(how='all',axis=1,inplace=True)
|
||||
drop_cols = [
|
||||
'regne','phylum','classe','ordre','famille','group1_inpn','group2_inpn','cd_ref',
|
||||
'altitude_z','longitude_x','latitude_y','geom','nom_commune',
|
||||
'etude','lot_donnee','observateurs','structures'
|
||||
]
|
||||
d = df.columns[df.columns.isin(drop_cols)]
|
||||
df.drop(columns=d,inplace=True)
|
||||
if df.shape[0] > 50000:
|
||||
from math import ceil
|
||||
loop = ceil(df.shape[0]/50000)
|
||||
i = 0
|
||||
for j in range(1,loop+1):
|
||||
p = path_name.replace('.csv','_%s.csv'%j)
|
||||
jj = j*50000
|
||||
df.iloc[i:jj].to_csv(p,index=False)
|
||||
i = jj
|
||||
else:
|
||||
df.to_csv(path_name,index=False)
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
test = False
|
||||
PATH = '/home/colas/Documents/9_PROJETS/4_SICEN/GN_MIGRATION/'
|
||||
al = flore.all()
|
||||
va = flore.vasculaire()
|
||||
br = flore.bryophyte()
|
||||
ch = flore.characee()
|
||||
|
||||
export_csv(al,PATH+'FLORE/all.csv')
|
||||
# export_csv(va[va.effectif< va.effectif_max].copy(),PATH+'FLORE/test_eff_vasculaire.csv')
|
||||
export_csv(va,PATH+'FLORE/vasculaire.csv')
|
||||
export_csv(br,PATH+'FLORE/bryophyte.csv')
|
||||
export_csv(ch,PATH+'FLORE/characee.csv')
|
||||
al.shape[0] - (va.shape[0]+br.shape[0]+ch.shape[0])
|
||||
|
||||
fa = faune.all()
|
||||
fia = faune.invertebre.all()
|
||||
fva = faune.vertebre.all()
|
||||
fa.shape[0] - (fia.shape[0] + fva.shape[0])
|
||||
if test :
|
||||
fa[~fa.id_obs.isin([*fia.id_obs,*fva.id_obs])]
|
||||
|
||||
ca = champignon()
|
||||
|
||||
vm = _vm_synthese_observations_()
|
||||
date_cols = vm.columns[vm.columns.str.contains('date')]
|
||||
mtd = vm[['lot_donnee','protocole','etude']].drop_duplicates()
|
||||
|
||||
vm.shape[0] - (al.shape[0]+fa.shape[0]+ca.shape[0])
|
||||
if test :
|
||||
vm[~vm.id_obs.isin([*al.id_obs,*fa.id_obs,*ca.id_obs])]
|
||||
|
||||
|
||||
# Sub verification
|
||||
fiod = faune.invertebre.odonate()
|
||||
fior = faune.invertebre.orthoptere()
|
||||
file = faune.invertebre.lepidoptere()
|
||||
fimo = faune.invertebre.mollusque()
|
||||
ficr = faune.invertebre.crustacees()
|
||||
fias = faune.invertebre.ascomycete()
|
||||
fiau = faune.invertebre.autre()
|
||||
fia.shape[0] - (
|
||||
fiod.shape[0] + fior.shape[0] + file.shape[0] + fimo.shape[0] + ficr.shape[0] + fias.shape[0] + fiau.shape[0]
|
||||
)
|
||||
if test :
|
||||
fia[~fia.id_obs.isin([*fiod.id_obs,*fior.id_obs,*file.id_obs,*fimo.id_obs,*ficr.id_obs,*fias.id_obs,*fiau.id_obs])]
|
||||
|
||||
fvma = faune.vertebre.mammifere()
|
||||
fvch = faune.vertebre.chiroptere()
|
||||
fvoi = faune.vertebre.oiseau()
|
||||
fvre = faune.vertebre.reptile()
|
||||
fvam = faune.vertebre.amphibien()
|
||||
fvpo = faune.vertebre.poisson()
|
||||
fva.shape[0] - (
|
||||
fvma.shape[0] + fvch.shape[0] + fvoi.shape[0] + fvre.shape[0] + fvam.shape[0] + fvpo.shape[0]
|
||||
)
|
||||
if test :
|
||||
fva[~fva.id_obs.isin([*fvma.id_obs,*fvch.id_obs,*fvoi.id_obs,*fvre.id_obs,*fvam.id_obs,*fvpo.id_obs])]
|
||||
|
||||
@@ -0,0 +1,860 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
|
||||
from pycen import update_to_sql, con_sicen,con_gn
|
||||
import geopandas as gpd
|
||||
import os
|
||||
|
||||
DIR = '/home/colas/Documents/9_PROJETS/4_SICEN/GN_MIGRATION'
|
||||
|
||||
def form_cdnom(data):
|
||||
dict_cdnom = {
|
||||
116744:521658, # Flore : Quercus petraea
|
||||
}
|
||||
data.replace({'cd_nom':dict_cdnom},inplace=True)
|
||||
data.replace({'cd_ref':dict_cdnom},inplace=True)
|
||||
|
||||
def form_complx_grp(data):
|
||||
dict_cdnom = {
|
||||
9999005:105817, # Leucanthemum vulgare (#groupe)
|
||||
9999014:119097, # Rubus fruticosus (#groupe)
|
||||
9999017:717630, # Taraxacum officinale (#groupe)
|
||||
9999019:126573, # Thymus serpyllum (#groupe)
|
||||
9999020:129298, # Vicia sativa (#groupe)
|
||||
9999023:188772, # Acrocephalus palustris / scirpaceus (#complexe)
|
||||
9999031:441709, # Cairina moschata f. domestica (#forme)
|
||||
9999033:190350, # Carduelis flammea flammea / cabaret / Carduelis hornemanni (#complexe)
|
||||
9999037:191029, # Colias alfacariensis / hyale (#complexe)
|
||||
9999041:4503, # Corvus corone corone / cornix (#complexe)
|
||||
9999042:186239, # Eptesicus / Nyctalus sp. (#complexe)
|
||||
9999046:192539, # Felis silvestris / catus (#complexe)
|
||||
9999050:193993, # Leptidea sinapis / reali (#complexe)
|
||||
9999054:194481, # Martes martes / foina (#complexe)
|
||||
9999057:195005, # Myotis myotis / blythii (#complexe)
|
||||
9999063:444436, # Pelophylax kl. esculentus / lessonae (#complexe)
|
||||
9999064:4280, # Phylloscopus collybita tristis / "abietinus" (#complexe)
|
||||
9999066:196980, # Pyrgus malvae / malvoides (#complexe)
|
||||
9999074:197040, # Rana dalmatina / temporaria (#complexe)
|
||||
9999075:196296, # Pipistrellus nathusii / kuhlii (#complexe)
|
||||
9999080:194357, # Lysandra coridon / hispana (#complexe)
|
||||
9999082:195005, # Myotis daubentonii / Myotis mystacinus (#complexe)
|
||||
9999083:699094, # Pipistrellus / Miniopterus (#complexe)
|
||||
}
|
||||
lst_cdnom_old = [*dict_cdnom.keys()]
|
||||
|
||||
if data.cd_nom.isin(lst_cdnom_old).any():
|
||||
data.loc[data.cd_nom.isin(lst_cdnom_old),'complexe_groupe'] = data[data.cd_nom.isin(lst_cdnom_old)].nom_complet
|
||||
data.replace({'cd_nom':dict_cdnom},inplace=True)
|
||||
|
||||
sql = 'SELECT cd_nom, nom_complet nom_new FROM taxonomie.taxref where cd_nom in {}'.format(tuple(dict_cdnom.values()))
|
||||
tax = gpd.pd.read_sql(sql, con_gn)
|
||||
data = data.merge(tax,how='left',on='cd_nom')
|
||||
data.loc[data.nom_new.notna(),'nom_latin'] = data[data.nom_new.notna()].nom_new
|
||||
data.drop(columns='nom_new',inplace=True)
|
||||
|
||||
def correct_taxonomie(data):
|
||||
dict_taxo = {
|
||||
444440:'Pelophylax kl\. esculentus \(Linnaeus, 1758\)'
|
||||
}
|
||||
for k,v in dict_taxo.items():
|
||||
is_saisie = data.rmq_observation.str.contains(v,na=False)
|
||||
if not is_saisie.any():
|
||||
continue
|
||||
data.loc[is_saisie,'nom_latin'] = v
|
||||
data.loc[is_saisie,'cd_nom'] = k
|
||||
data.loc[is_saisie,'cd_ref'] = k
|
||||
|
||||
def form_precision(data):
|
||||
dict_pre = {
|
||||
'GPS':0,
|
||||
'0 à 10m':10,
|
||||
'10 à 100m':100,
|
||||
'100 à 500m':500,
|
||||
'lieu-dit':750,
|
||||
'commune':None,
|
||||
}
|
||||
is_com = data.precision =='commune'
|
||||
rmq_null = data.rmq_localisation.isnull()
|
||||
data.loc[is_com&rmq_null,'rmq_localisation'] = 'Localisation : commune'
|
||||
data.loc[is_com&~rmq_null,'rmq_localisation'] = 'Localisation : commune ;'+data[is_com].rmq_localisation
|
||||
data.replace({'precision':dict_pre},inplace=True)
|
||||
data.precision = data.precision.astype('Int64')
|
||||
|
||||
|
||||
def form_effectif(data):
|
||||
eff_notna = data.effectif.notna()
|
||||
efmin_notna = data.effectif_min.notna()
|
||||
efmax_isna = data.effectif_max.isna()
|
||||
data.loc[(~eff_notna)&efmin_notna,'effectif'] = data[(~eff_notna)&efmin_notna].effectif_min
|
||||
data.loc[(~eff_notna)&(~efmax_isna),'effectif'] = data[(~eff_notna)&(~efmax_isna)].effectif_max
|
||||
# data.loc[efmax_isna,'effectif_max'] = data[efmax_isna].effectif
|
||||
data.effectif = data.effectif.astype('Int64')
|
||||
data.effectif_max = data.effectif_max.astype('Int64')
|
||||
|
||||
t1 = data.effectif_min < data.effectif
|
||||
data.loc[t1,'effectif_max'] = data[t1].effectif
|
||||
data.loc[t1,'effectif'] = data[t1].effectif_min
|
||||
|
||||
|
||||
# Est trop incertain... Les effectifs récupérés sont < aux effectifs déclarés
|
||||
# ==> Incompréhansion, Ne sera pas utilisé
|
||||
def recup_faune_eff(data):
|
||||
# Récupération des effectifs standardisés du champ rmq_observation
|
||||
ismixte = data.rmq_observation.str.contains('\\d.00 Mâle\(s\) et \\d.00 Femelle\(s\)',na=False)
|
||||
ismal = data.rmq_observation.str.contains('\\d.00 Mâle\(s\) et \? Femelle\(s\)',na=False)
|
||||
isfem = data.rmq_observation.str.contains('\? Mâle\(s\) et \\d.00 Femelle\(s\)',na=False)
|
||||
data.loc[ismixte,'extract_eff'] = (data[ismixte].rmq_observation
|
||||
.str.extract('(\d+).00 Mâle\(s\) et (\d+).00 Femelle\(s\)').astype(int).sum(axis=1))
|
||||
data.loc[ismal,'extract_eff'] = (data[ismal].rmq_observation
|
||||
.str.extract('(\d+).00 Mâle\(s\) et \? Femelle\(s\)').astype(int).sum(axis=1))
|
||||
data.loc[isfem,'extract_eff'] = (data[isfem].rmq_observation
|
||||
.str.extract('\? Mâle\(s\) et (\d+).00 Femelle\(s\)').astype(int).sum(axis=1))
|
||||
|
||||
extr = data.extract_eff.notna()
|
||||
extr_sup = data.effectif < data.extract_eff
|
||||
data[extr_sup]
|
||||
age_na = data.age_faune.isna()
|
||||
no_0 = data.extract_eff > 0
|
||||
if any(age_na&no_0&ismixte):
|
||||
data.loc[age_na&no_0&ismixte,'age_faune'] = 'mixte'
|
||||
if any(age_na&no_0&ismal):
|
||||
data.loc[age_na&no_0&ismal,'age_faune'] = 'male'
|
||||
if any(age_na&no_0&isfem):
|
||||
data.loc[age_na&no_0&isfem,'age_faune'] = 'femelle'
|
||||
|
||||
def form_date(data):
|
||||
cols = data.columns[data.columns.str.contains('date')]
|
||||
for col in cols:
|
||||
data.loc[data[col].isna(),col] = None
|
||||
|
||||
def recup_stadevie(data):
|
||||
dict_repro = {
|
||||
'ODO_Exuvie/émergence':'Exuvie/émergence',
|
||||
'ODO_Immature':'Immature',
|
||||
**dict.fromkeys(['ODO_Mâles+Femelles','ODO_Tandem','ODO_Territorial','ODO_Ponte'],'Adulte'),
|
||||
**dict.fromkeys(['CHIR_Indéterminé','ODO_Indéterminé'],'Indéterminé')
|
||||
}
|
||||
if 'age_faune' in data.columns:
|
||||
age_isna = data.age_faune.isna()
|
||||
age_inrmq = data.rmq_observation.str.contains('Stade de vie')
|
||||
data.loc[age_isna&age_inrmq,'age_faune'] = (data[age_isna&age_inrmq].rmq_observation
|
||||
.str.split('Stade de vie :')
|
||||
.str[1]
|
||||
.str.split('|')
|
||||
.str[0]
|
||||
.str.strip())
|
||||
double_info = data.age_faune.isin(['Imago, adulte', 'Nymphe, immature'])
|
||||
d2 = data[double_info].copy()
|
||||
|
||||
data.age_faune = data.age_faune.str.replace('Imago, adulte','Imago')
|
||||
data.age_faune = data.age_faune.str.replace('Nymphe, immature','Nymphe')
|
||||
|
||||
d2.age_faune = d2.age_faune.str.replace('Imago, adulte','Adulte')
|
||||
d2.age_faune = d2.age_faune.str.replace('Nymphe, immature','Immature')
|
||||
d2.id_obs = ('9999'+d2.id_obs.astype(str)).astype(int)
|
||||
|
||||
data = gpd.pd.concat([data,d2])
|
||||
|
||||
lst_age = [*dict_repro.keys()]
|
||||
age_isna = data.age_faune.isna()
|
||||
t1 = age_isna&(data.reprostatut_faune.isin(lst_age))
|
||||
data.loc[t1,'age_faune'] = data[t1].reprostatut_faune.replace(dict_repro)
|
||||
|
||||
t2 = data.age_faune=='Exuvie/émergence'
|
||||
is_exuvie = (data.rmq_observation
|
||||
.replace(['é','E','É'],'e',regex=True)
|
||||
.str.contains('exuvie',na=False))
|
||||
is_emerge = (data.rmq_observation
|
||||
.replace(['é','E','É'],'e',regex=True)
|
||||
.str.contains('emerge',na=False))
|
||||
rmq_isna = data.rmq_observation.isna()
|
||||
|
||||
data.loc[is_exuvie&~is_emerge&t2,'age_faune'] = 'Exuvie'
|
||||
data.loc[~is_exuvie&is_emerge&t2,'age_faune'] = 'Emergent'
|
||||
data.loc[is_exuvie&is_emerge&t2,'age_faune'] = 'Exuvie'
|
||||
data.loc[is_exuvie&is_emerge&rmq_isna&t2,'age_faune'] = 'Emergent'
|
||||
|
||||
t3 = data.age_faune=='Oeuf/ponte/larve/nymphe/chenille...'
|
||||
# Odonate
|
||||
is_odo = data.ordre == 'Odonata'
|
||||
data.loc[t3&is_odo,'age_faune'] = 'Exuvie'
|
||||
# Amphibiens
|
||||
is_amphi = data.group2_inpn == 'Amphibiens'
|
||||
is_larve = data.rmq_observation.str.contains('larve',na=False)
|
||||
is_ponte = data.rmq_observation.str.contains('ponte|Ponte',na=False)
|
||||
is_tetar = data.rmq_observation.replace(['ê','é'],'e',regex=True).str.contains('tetard',na=False)
|
||||
is_urode = data.ordre=='Urodela'
|
||||
is_anure = data.ordre=='Anura'
|
||||
rmq_na = data.rmq_observation.isna()
|
||||
data.loc[t3&is_amphi&is_larve,'age_faune'] = 'Larve'
|
||||
data.loc[t3&is_amphi&is_ponte,'age_faune'] = 'Oeufs'
|
||||
data.loc[t3&is_amphi&is_tetar,'age_faune'] = 'Têtard'
|
||||
data.loc[t3&is_amphi&is_urode&rmq_na,'age_faune'] = 'Larve'
|
||||
data.loc[t3&is_amphi&is_anure&rmq_na,'age_faune'] = 'Ponte'
|
||||
# Corresction de 3 données de Nicolas Biron
|
||||
t3 = data.age_faune=='Oeuf/ponte/larve/nymphe/chenille...'
|
||||
data.loc[t3&(is_urode|is_anure),'age_faune'] = 'Ponte'
|
||||
# Correction de la dernière donnée restante (Marjorie Simean)
|
||||
t3 = data.age_faune=='Oeuf/ponte/larve/nymphe/chenille...'
|
||||
data.loc[t3,'age_faune'] = 'Oeufs'
|
||||
|
||||
# Définition de l'objet du dénombrement si ponte
|
||||
if 'obj_denombre' not in data.columns:
|
||||
data['obj_denombre'] = None
|
||||
isnot_ponte = ~data.rmq_observation.str.contains('nbre pontes:0',na=False)
|
||||
data.loc[is_amphi&is_ponte&isnot_ponte,'obj_denombre'] = 'Ponte'
|
||||
|
||||
# Récup stade_vie derrirère le terme @NSA
|
||||
age_isna = data.age_faune.isna()
|
||||
as_nsa = data.rmq_observation.str.contains('@NSA',na=False)
|
||||
# data.loc[age_isna&as_nsa,'test'] = (data[age_isna&as_nsa].rmq_observation
|
||||
# .str.split('@NSA :|@NSA:')
|
||||
# .str[1]
|
||||
# .str.split('/|;')
|
||||
# .str[0]
|
||||
# .str.strip())
|
||||
data.loc[age_isna&as_nsa,'age_faune'] = (data[age_isna&as_nsa].rmq_observation
|
||||
.str.split('@NSA :|@NSA:')
|
||||
.str[1]
|
||||
.str.split('/|;')
|
||||
.str[0]
|
||||
.str.strip()
|
||||
.str.lower()
|
||||
.str.extract('(imm|juv|ad|jeune|larve)')[0])
|
||||
|
||||
# Récup stade_vie derrirère le terme Commentaires
|
||||
age_isna = data.age_faune.isna()
|
||||
as_com = data.rmq_observation.str.contains('Commentaires : \\d',na=False)
|
||||
data.loc[age_isna&as_com,'age_faune'] = (data[age_isna&as_com].rmq_observation
|
||||
.str.split('[,(.]|Code Atlas')
|
||||
.str[0].str.strip()
|
||||
.str.lower()
|
||||
.replace(
|
||||
{'femelles':'femelle','mâles':'mâle','adultes':'adulte',
|
||||
'jeunes':'jeune','juvéniles':'juvénile','larves':'larve',
|
||||
'couples':'couple','x':''},regex=True)
|
||||
.str.extract('[Commentaires : \d+ ](jeune volant|juvénile|juv|couple|adulte|ad|crapelet|femelle adulte|larve|mâle juvénile|mâle adulte|subadulte|larve \/ têtard)$')[0])
|
||||
age_isna = data.age_faune.isna()
|
||||
data.loc[age_isna&as_com,'age_faune'] = (data[age_isna&as_com].rmq_observation
|
||||
.str.split('[,(.]|Code Atlas')
|
||||
.str[0].str.strip()
|
||||
.str.lower()
|
||||
.replace(
|
||||
{'femelles':'femelle','mâles':'mâle','adultes':'adulte',
|
||||
'jeunes':'jeune','juvéniles':'juvénile','larves':'larve',
|
||||
'têtards':'têtard','couples':'couple','x':''},regex=True)
|
||||
.str.extract('[Commentaires : \d+](jeune volant|juvénile|juv|couple|adulte|ad|crapelet|femelle adulte|larve|mâle adulte|subadulte|larve / têtard)$')[0])
|
||||
|
||||
# Précision de larve / têtard
|
||||
is_impreci = data.age_faune=='larve / têtard'
|
||||
data.loc[is_impreci&is_anure,'age_faune'] = 'têtard'
|
||||
data.loc[is_impreci&is_urode,'age_faune'] = 'larve'
|
||||
|
||||
data.age_faune = (data.age_faune
|
||||
.replace({
|
||||
'ad':'Adulte',
|
||||
'Oeufs':'Oeuf',
|
||||
'imm':'Immature',
|
||||
'jeune':'Jeune',
|
||||
'juv':'Juvénile',
|
||||
'subadulte':'Sub-adulte',
|
||||
'Subadulte':'Sub-adulte',})
|
||||
.str.capitalize()
|
||||
)
|
||||
|
||||
sex_isna = data.sexe_faune.isna()
|
||||
is_male = data.age_faune.str.contains('Mâle',na=False)
|
||||
is_feml = data.age_faune.str.contains('Femelle',na=False)
|
||||
is_copl = data.age_faune.str.contains('Couple',na=False)
|
||||
is_immat = data.age_faune.str.contains('Immature',na=False)
|
||||
data.loc[sex_isna&is_male,'sexe_faune'] = 'Mâle'
|
||||
data.loc[sex_isna&is_feml,'sexe_faune'] = 'Femelle'
|
||||
data.loc[sex_isna&is_copl,'sexe_faune'] = 'Couple'
|
||||
data.loc[sex_isna&is_immat,'sexe_faune'] = 'Immature'
|
||||
|
||||
return data
|
||||
|
||||
def complete_objdenomb(data):
|
||||
asnot_denomb = data.obj_denombre.isna()
|
||||
isnot_dead = data.etat_bio != 'mort'
|
||||
as_statbio = data.statut_bio.notna()
|
||||
as_compt = data.comportement.notna()
|
||||
as_atlas = data.code_atlas.notna()
|
||||
as_age = data.age_faune.notna()
|
||||
as_sexe = data.sexe_faune.notna()
|
||||
as_repro = data.reprostatut_faune.notna()
|
||||
as_determ = data.determination.notna()
|
||||
data.loc[asnot_denomb&isnot_dead&(as_statbio|as_compt|as_atlas|as_age|as_sexe|as_repro|as_determ),'obj_denombre'] = 'individu'
|
||||
return data
|
||||
|
||||
def recup_comptmt(data):
|
||||
if 'reprostatut_faune' in data.columns:
|
||||
# Récupération du comportement après `Comp. ind` dans la colonne `rmq_observation`
|
||||
compt_isna = data.reprostatut_faune.isna()
|
||||
in_rmq = data.rmq_observation.str.contains('Comp. ind')
|
||||
data.loc[compt_isna&in_rmq,'comportement'] = (data[compt_isna&in_rmq].rmq_observation
|
||||
.str.split('Comp. ind. :')
|
||||
.str[1]
|
||||
.str.split('|')
|
||||
.str[0]
|
||||
.str.strip())
|
||||
# Récupération du comportement dans la colonne `reprostatut_faune`
|
||||
sex_isna = data.sexe_faune.isna()
|
||||
comp_isna = data.comportement.isna()
|
||||
lst_comp_inv = [
|
||||
'ODO_Tandem','ODO_Territorial','INV_Accouplement','INV_Chant (orthoptères)','ODO_Ponte']
|
||||
lst_comp_ver = ['OIS_Reproduction certaine (patrim)','MAM_Fécès - épreinte - urine', 'OIS_Nid avec jeune(s)',
|
||||
'OIS_Couvaison', 'CHIR_Hivernant', 'AMP_Mise bas - ponte','CHIR_Lethargie', 'OIS_Chant', # 'AMP_Indéterminé',
|
||||
'AMP_Chant','CHIR_Estivant', 'CHIR_Transit', 'OIS_Reproduction possible',
|
||||
'MAM_Alarme', 'AMP_Accouplement', 'OIS_Fuite - envol','OIS_Accouplement']
|
||||
if data.reprostatut_faune.isin(lst_comp_inv).any():
|
||||
t3 = (~compt_isna)&comp_isna&sex_isna&(data.reprostatut_faune.isin(lst_comp_inv))
|
||||
elif data.reprostatut_faune.isin(lst_comp_ver).any():
|
||||
t3 = (~compt_isna)&comp_isna&(data.reprostatut_faune.isin(lst_comp_ver))
|
||||
data.loc[t3,'comportement'] = data[t3].reprostatut_faune
|
||||
|
||||
# Récupération du comportement après `Comp. ind` dans la colonne `rmq_observation`
|
||||
# dictionnaire des transports
|
||||
dict_transport = {
|
||||
**dict.fromkeys(['transport de branche','transport de matériaux','transport de brindille','transporte une branche',"transport d'une branche",'transportant des matériaux','transportant une branche','transporte des feuilles'],'transport matériaux'),
|
||||
**dict.fromkeys(['transport de nourriture','transport nourriture','transport de poisson','transportant nourriture','Transporte une proie','transport de proie'],'transport nourriture'),
|
||||
}
|
||||
lst_rmq_comp = "chant|houspille|en chasse|{}|en vol|en survol|posé|parade|s'envole|se nourrit|se pose|tambourinage|vol migratoire|hivernage|nourrissage|couveur|nourrit|cht|cris|estivant|hivernant|Construction de nid|alarme|migration active".format('|'.join(dict_transport.keys()))
|
||||
comp_isna = data.comportement.isna()
|
||||
in_rmq = data.rmq_observation.str.contains('[%s]'%lst_rmq_comp,case=False,na=False)
|
||||
data.loc[comp_isna&in_rmq,'comportement'] = (data[comp_isna&in_rmq].rmq_observation
|
||||
.str.lower()
|
||||
.str.extract('(%s)'%lst_rmq_comp)[0]
|
||||
.replace(dict_transport))
|
||||
|
||||
comp_isna = data.comportement.isna()
|
||||
envol = data.rmq_observation.str.contains('Statut:Passage.')
|
||||
if any(comp_isna&envol):
|
||||
data.loc[comp_isna&envol,'comportement'] = 'en vol'
|
||||
|
||||
comp_isna = data.comportement.isna()
|
||||
data.loc[comp_isna,'comportement'] = (data[comp_isna].rmq_observation
|
||||
.replace({'x',''},regex=True)
|
||||
.str.lower()
|
||||
.str.extract('[détails : \d+ ]\((en transit)\)')[0])
|
||||
|
||||
def _get_gn_atlas():
|
||||
sch = 'ref_nomenclatures'
|
||||
tab = ['t_nomenclatures','bib_nomenclatures_types']
|
||||
bib_mnemo='CODE_ATLAS'
|
||||
sql = """
|
||||
SELECT
|
||||
a.id_nomenclature,
|
||||
a.cd_nomenclature,
|
||||
a.mnemonique mnemo,
|
||||
a.label_default as label,
|
||||
a.label_fr,
|
||||
a.definition_default definition,
|
||||
a.definition_fr def_fr,
|
||||
b.mnemonique bib_mnemo,
|
||||
b.label_default bib_label,
|
||||
b.definition_default bib_def,
|
||||
a.active
|
||||
FROM {sch}.{tab0} a
|
||||
JOIN {sch}.{tab1} b USING (id_type)
|
||||
""".format(sch=sch,tab0=tab[0],tab1=tab[1])
|
||||
if bib_mnemo is not None:
|
||||
sql += " WHERE "
|
||||
sql += "b.mnemonique = '%s'"%bib_mnemo
|
||||
return gpd.pd.read_sql_query(sql,con_gn)
|
||||
|
||||
|
||||
def recup_atlas(data):
|
||||
is_atlas = data.rmq_observation.str.contains('Code atlas',na=False)
|
||||
# is_atlas = data.rmq_observation.str.contains('Code atlas :',na=False)
|
||||
|
||||
if any(is_atlas):
|
||||
atlas = _get_gn_atlas()
|
||||
dict_code = dict(zip(atlas.cd_nomenclature.astype(int).astype(str),atlas.label_fr))
|
||||
dict_label = dict(zip(atlas.label.str.removesuffix('.'),atlas.label_fr))
|
||||
dict_autre = {
|
||||
'Mâle chanteur (ou cris de nidification) ou tambourinage en période de reproduction':'Mâle chanteur présent en période de nidification, cris nuptiaux ou tambourinage entendus, mâle vu en parade',
|
||||
'Jeunes fraîchement envolés (espèces nidicoles) ou poussins (espèces nidifuges)':'Jeunes en duvet ou jeunes venant de quitter le nid et incapables de soutenir le vol sur de longues distances'
|
||||
}
|
||||
data.loc[is_atlas,'code_atlas'] = (data[is_atlas].rmq_observation
|
||||
.str.split('Code atlas')
|
||||
.str[1]
|
||||
# .str.split('\||@|;|\(')
|
||||
.str.split(r'[|@;]')
|
||||
.str[0]
|
||||
.replace({
|
||||
**dict.fromkeys([':','Nidification possible \(','Nidification certaine \(','Nidification probable \('],''),
|
||||
'\s+':' ',
|
||||
' - ':'; '},regex=True)
|
||||
.str.strip()
|
||||
.str.replace('nuptial parades','nuptial: parades')
|
||||
.str.removesuffix(')')
|
||||
.replace('',None)
|
||||
.replace(dict_code)
|
||||
.replace(dict_autre)
|
||||
.replace(dict_label))
|
||||
|
||||
data.loc[is_atlas,'rmq1'] = (data[is_atlas].rmq_observation
|
||||
.str.split('Code atlas')
|
||||
.str[0]
|
||||
.str.strip()
|
||||
.replace(dict.fromkeys(['Commentaires :','Remarques :','Détails : @','','nan'],None))
|
||||
)
|
||||
data.loc[is_atlas,'rmq2'] = (data[is_atlas].rmq_observation
|
||||
.str.split('Code atlas')
|
||||
.str[1]
|
||||
.str.split(r'[|@;]')
|
||||
.str[1]
|
||||
.str.strip()
|
||||
.str.removesuffix(';')
|
||||
.str.strip()
|
||||
.replace(dict.fromkeys(['','nan'],None))
|
||||
)
|
||||
data.loc[is_atlas,'rmq_observation'] = [
|
||||
' ; '.join([x,y]) if not gpd.pd.isna(y) and not gpd.pd.isna(x)
|
||||
else y if gpd.pd.isna(x)
|
||||
else x
|
||||
for x,y in zip(data[is_atlas].rmq1,data[is_atlas].rmq2)
|
||||
]
|
||||
data.rmq_observation.replace('nan',None,inplace=True)
|
||||
data.drop(columns=['rmq1','rmq2'],inplace=True)
|
||||
|
||||
atlas_isna = data.code_atlas.isna()
|
||||
rmq_atlas = {
|
||||
'2':"Observation de l'espèce pendant la période de nidification dans un biotope adéquat",
|
||||
'2':"Observation de l'espèce pendant la période de nidification",
|
||||
'4':'Couple pendant la période de nidification dans un biotope adéquat',
|
||||
}
|
||||
for k,v in rmq_atlas.items():
|
||||
as_atlas = data.rmq_observation.str.contains(v,na=False)
|
||||
data.loc[atlas_isna&as_atlas,'code_atlas'] = dict_code[k]
|
||||
|
||||
# Récupération du code atlas 99
|
||||
for i in ['99','1.00','2','3.00','3 ','4 ','4.00','5.00','5 ',
|
||||
'6','7','8','9.00','9 ','10','11 ','12 ','12.00','13',
|
||||
'14.00','14 ','15.00','16','17','18','19']:
|
||||
is_atlas = data.rmq_observation.str.contains('Code Atlas : %s'%i,na=False)
|
||||
atlas_isna = data.code_atlas.isna()
|
||||
if is_atlas.any():
|
||||
data.loc[is_atlas&atlas_isna,'code_atlas'] = str(int(float('%s'%i)))
|
||||
data.loc[is_atlas&atlas_isna,'code_atlas'] = data[is_atlas&atlas_isna].code_atlas.replace(dict_code)
|
||||
|
||||
# définition du code_atlas par le statut repro
|
||||
lst_repro = [
|
||||
'Reproduction possible','Reproduction probable','Reproduction confirmée','Nicheur possible'
|
||||
]
|
||||
|
||||
atlas_isna = data.code_atlas.isna()
|
||||
is_aves = data.group2_inpn == 'Oiseaux'
|
||||
is_repro = data.statut_bio.isin(lst_repro)
|
||||
if any(is_aves&is_repro&atlas_isna):
|
||||
data.loc[is_aves&is_repro&atlas_isna,'code_atlas'] = (data[is_aves&is_repro&atlas_isna].statut_bio
|
||||
.replace(['Reproduction','Nicheur'],'Nidification')
|
||||
.replace(dict_label))
|
||||
|
||||
# définition du code atlas par le statut repro
|
||||
atlas_isna = data.code_atlas.isna()
|
||||
dict_repro = {
|
||||
**dict.fromkeys(['couvaison','couveur','ois_couvaison'],'18'),
|
||||
**dict.fromkeys(['accouplement','ois_accouplement','parade'],'6'),
|
||||
**dict.fromkeys(['alarme'],'8'),
|
||||
**dict.fromkeys(['chant','cht','ois_chant'],'3'),
|
||||
**dict.fromkeys(['nourrissage','nourrissage des jeunes'],'16'),
|
||||
**dict.fromkeys(['ois_nid avec jeune(s)'],'19'),
|
||||
**dict.fromkeys(['ois_reproduction certaine (patrim)'],'50'),
|
||||
**dict.fromkeys(['ois_reproduction possible'],'30'),
|
||||
**dict.fromkeys(['transport matériaux'],'10'),
|
||||
}
|
||||
for k,v in dict_repro.items():
|
||||
is_repro = data.comportement.str.lower()==k
|
||||
if any(is_repro&atlas_isna):
|
||||
data.loc[is_repro&is_aves&atlas_isna,'code_atlas'] = dict_code[v]
|
||||
|
||||
# Suppression des codes atlas érronés
|
||||
is_dead = data.etat_bio=='mort'
|
||||
atlas_notna = data.code_atlas.notna()
|
||||
if any(is_dead&atlas_notna):
|
||||
data.loc[is_dead&atlas_notna,'code_atlas'] = None
|
||||
|
||||
|
||||
def form_observator(data):
|
||||
return data.observateurs_v2.replace({
|
||||
' \(CEN Isère\) \(CEN Isère\)':'',
|
||||
' \(GRPLS\) \(GRPLS\)':'',
|
||||
' \(CD Isère\) \(CD Isère\)':'',
|
||||
' \(BIOTOPE\) \(BIOTOPE\)':'',
|
||||
' \(Ecosphère\) \(Ecosphère\)':'',
|
||||
' \(FMBDS\) \(FMBDS\)':'',
|
||||
' \(Lo Parvi\) \(Lo Parvi\)':'',
|
||||
' \(LPO Rhône\) \(LPO Rhône\)':'',
|
||||
' \(Personnel\) \(Personnel\)':'',
|
||||
' \(ONCFS\) \(ONCFS\)':'',
|
||||
' \(Nature et Vie Sociale\) \(Nature et Vie Sociale\)':'',
|
||||
' \(Conservatoire d’Espaces Naturels Isère\) \(Conservatoire d’Espaces Naturels Isère\)':'',
|
||||
' \(Société Batrachologique de France\) \(Société Batrachologique de France\)':'',
|
||||
'CEN Isère':'Conservatoire d’Espaces Naturels Isère',
|
||||
},regex=True)
|
||||
|
||||
def form_coord(data):
|
||||
t1 = data.longitude_x > 10
|
||||
t2 = data.latitude_y > 50
|
||||
data.loc[t1,'longitude_x'] = data[t1].to_crs(epsg=4326).geom.x
|
||||
data.loc[t2,'latitude_y'] = data[t2].to_crs(epsg=4326).geom.y
|
||||
|
||||
def form_rmqobs(data):
|
||||
data.rmq_observation.replace({
|
||||
'Commentaires : Détails :':'Détails :',
|
||||
'Commentaires : , localisation :':'localisation :',
|
||||
},inplace=True,regex=True)
|
||||
|
||||
def recup_determ(data):
|
||||
determ_isna = data.determination.isna()
|
||||
is_statut = data.rmq_observation.str.contains('Indice de présence')
|
||||
if is_statut.any():
|
||||
data.loc[determ_isna&is_statut,'determination'] = 'Indice de présence'
|
||||
data.loc[determ_isna&is_statut,'preuve_detrm'] = (data[determ_isna&is_statut].rmq_observation
|
||||
.str.split('Indice de présence')
|
||||
.str[1]
|
||||
.str.split('|')
|
||||
.str[0]
|
||||
.str.strip()
|
||||
.str.extract('.*\((.*)\).*')[0]
|
||||
)
|
||||
|
||||
determ_isna = data.determination.isna()
|
||||
data.loc[determ_isna,'determination'] = (data[determ_isna].rmq_observation
|
||||
.str.extract('.*\((en main|observation indirecte|vu|contact auditif|analyse de pelotes)\).*')[0]
|
||||
)
|
||||
|
||||
# determ_isna = data.determination.isna()
|
||||
# in_hand = data.rmq_observation.str.contains('\(en main\)')
|
||||
# if any(determ_isna&in_hand):
|
||||
# data.loc[determ_isna&in_hand,'determination'] = 'en main'
|
||||
|
||||
determ_isna = data.determination.isna()
|
||||
vu = data.rmq_observation.str.contains('Individu vu|Vu vivant|Commentaires \: Vu')
|
||||
if any(determ_isna&vu):
|
||||
data.loc[determ_isna&vu,'determination'] = 'Individu vu'
|
||||
|
||||
determ_isna = data.determination.isna()
|
||||
entendu = data.rmq_observation.str.contains('Individu entendu')
|
||||
if any(determ_isna&entendu):
|
||||
data.loc[determ_isna&entendu,'determination'] = 'Individu entendu'
|
||||
|
||||
determ_isna = data.determination.isna()
|
||||
photo = data.rmq_observation.str.contains('piège photo')
|
||||
if any(determ_isna&photo):
|
||||
data.loc[determ_isna&photo,'determination'] = 'Observation par piège photographique'
|
||||
|
||||
def recup_statutbio(data):
|
||||
data['statut_bio'] = None
|
||||
is_stat1 = data.rmq_observation.str.contains('Caractéristique',na=False)
|
||||
if is_stat1.any():
|
||||
statbio_isna = data.statut_bio.isna()
|
||||
data.loc[statbio_isna&is_stat1,'statut_bio'] = (data[statbio_isna&is_stat1].rmq_observation
|
||||
.str.split('Caractéristique :')
|
||||
.str[1]
|
||||
.str.split('\|')
|
||||
.str[0]
|
||||
.str.strip()
|
||||
)
|
||||
is_stat2 = data.rmq_observation.str.contains('Statut',na=False)
|
||||
if is_stat2.any():
|
||||
statbio_isna = data.statut_bio.isna()
|
||||
data.loc[statbio_isna&is_stat2,'statut_bio'] = (data[statbio_isna&is_stat2].rmq_observation
|
||||
.str.split('Statut')
|
||||
.str[1]
|
||||
.str.strip()
|
||||
.str.removeprefix(':').str.strip()
|
||||
.str.split('\||/|;')
|
||||
.str[0]
|
||||
.str.strip()
|
||||
.str.extract('(Migrateur|Estivage|Reproduction probable|Reproduction certaine|Reproduction possible|Hivernant)')[0]
|
||||
)
|
||||
is_stat3 = data.rmq_observation.str.contains('Critere',na=False)
|
||||
if is_stat3.any():
|
||||
statbio_isna = data.statut_bio.isna()
|
||||
data.loc[statbio_isna&is_stat3,'statut_bio'] = (data[statbio_isna&is_stat3].rmq_observation
|
||||
.str.split('Critere')
|
||||
.str[1]
|
||||
.str.strip()
|
||||
.str.removeprefix(':').str.strip()
|
||||
.str.split('\||/|;')
|
||||
.str[0]
|
||||
.str.strip()
|
||||
.str.extract('(nicheurs possibles|reproducteur)')[0]
|
||||
)
|
||||
is_stat4 = data.rmq_observation.str.contains('Commentaires',na=False)
|
||||
if is_stat4.any():
|
||||
statbio_isna = data.statut_bio.isna()
|
||||
data.loc[statbio_isna&is_stat4,'statut_bio'] = (data[statbio_isna&is_stat4].rmq_observation
|
||||
.str.split('Commentaires')
|
||||
.str[1]
|
||||
.str.strip()
|
||||
.str.removeprefix(':').str.strip()
|
||||
.str.lower()
|
||||
.str.extract('(estivant|nicheur certain|nicheur probable|nicheur|semble nicher|niche|Reproduction probable|Reproduction certaine|Reproduction possible)')[0]
|
||||
)
|
||||
|
||||
def recup_etatbio(data):
|
||||
data['etat_bio'] = None
|
||||
# Récupération de la mortalité dans le champs rmq_observation
|
||||
is_dead1 = (data.rmq_observation
|
||||
.str.lower()
|
||||
.str.contains('individu mort|trouvé mort|mort : oui',na=False))
|
||||
if is_dead1.any():
|
||||
data.loc[is_dead1,'etat_bio'] = 'mort'
|
||||
|
||||
is_dead2 = data.comportement.str.lower()=='cadavre'
|
||||
if is_dead2.any():
|
||||
data.loc[is_dead2,'etat_bio'] = 'mort'
|
||||
|
||||
# nettoyage des comportements si individu mort
|
||||
is_dead = is_dead1|is_dead2
|
||||
compt_notna = data.comportement.notna()
|
||||
if any(is_dead&compt_notna):
|
||||
data.loc[is_dead&compt_notna,'comportement'] = None
|
||||
|
||||
# adaptation de la détermination
|
||||
detm_hand = data.determination=='en main'
|
||||
if any(detm_hand&is_dead):
|
||||
data.loc[detm_hand&is_dead,'comportement'] = 'cadavre en main'
|
||||
|
||||
def recup_sexefaune(data):
|
||||
is_male = data.rmq_observation.replace({'[x.0s]':''},regex=True).str.lower().str.contains('\\d mâle',na=False)
|
||||
is_feml = data.rmq_observation.replace({'[x.0s]':''},regex=True).str.lower().str.contains('\\d femelle',na=False)
|
||||
sexe_isna = data.sexe_faune.isna()
|
||||
data.loc[sexe_isna&is_male&~is_feml,'sexe_faune'] = 'mâle'
|
||||
data.loc[sexe_isna&~is_male&is_feml,'sexe_faune'] = 'femelle'
|
||||
data.loc[sexe_isna&is_male&is_feml,'sexe_faune'] = 'mixte'
|
||||
|
||||
def format_faune(data):
|
||||
recup_sexefaune(data)
|
||||
recup_comptmt(data)
|
||||
recup_etatbio(data)
|
||||
recup_statutbio(data)
|
||||
recup_atlas(data)
|
||||
res = recup_stadevie(data)
|
||||
return complete_objdenomb(res)
|
||||
|
||||
def format_data(data):
|
||||
form_rmqobs(data)
|
||||
data.observateurs_v2 = form_observator(data)
|
||||
form_precision(data)
|
||||
form_effectif(data)
|
||||
correct_taxonomie(data)
|
||||
form_cdnom(data)
|
||||
form_date(data)
|
||||
form_coord(data)
|
||||
form_complx_grp(data)
|
||||
recup_determ(data)
|
||||
return format_faune(data)
|
||||
|
||||
def _export(path_name, data, format='csv'):
|
||||
if format=='xlsx':
|
||||
data.to_excel(path_name+'.%s'%format)
|
||||
elif format=='csv':
|
||||
data.to_csv(path_name+'.%s'%format)
|
||||
else:
|
||||
raise('format non prévu')
|
||||
|
||||
def _filter_diffusable(path_name,data):
|
||||
if any(data.diffusable=='non'):
|
||||
no_diff = data[data.diffusable=='non'].copy()
|
||||
diff = data[data.diffusable=='oui'].copy()
|
||||
to_export = {
|
||||
path_name+'_diff':diff,
|
||||
path_name+'_no_diff':no_diff
|
||||
}
|
||||
else:
|
||||
to_export = {
|
||||
path_name:data
|
||||
}
|
||||
return to_export
|
||||
|
||||
|
||||
def _filter_uuid_sinp(to_export):
|
||||
res = {}
|
||||
for k,v in to_export.items():
|
||||
if 'unique_id_sinp' in v.columns:
|
||||
uuid_sinp = v[v.unique_id_sinp.notna()]
|
||||
not_uuid = v[v.unique_id_sinp.isna()]
|
||||
res |= {
|
||||
k+'_uuid':uuid_sinp,
|
||||
k+'_not_uuid':not_uuid
|
||||
}
|
||||
else:
|
||||
res |= {
|
||||
k:v
|
||||
}
|
||||
return res
|
||||
|
||||
def export(path_name, data, format='csv'):
|
||||
# Formatage des dates avant écriture
|
||||
detect_date = data.columns[data.columns.str.startswith('date')]
|
||||
data.loc[:,detect_date] = data[detect_date].astype(str)
|
||||
# Suppression de la colonne geom
|
||||
df = data.to_wkt().drop(columns='geom',errors='ignore')
|
||||
# Export
|
||||
_to_export = _filter_diffusable(path_name,df)
|
||||
to_export = _filter_uuid_sinp(_to_export)
|
||||
for k,v in to_export.items():
|
||||
if v.empty:
|
||||
continue
|
||||
_export(k,v,format)
|
||||
|
||||
def serena_rnngl_site():
|
||||
from sqlalchemy import create_engine # pour lecture de la bd
|
||||
from sqlalchemy.engine import URL
|
||||
from shapely.geometry import Polygon
|
||||
|
||||
usr = 'postgres'
|
||||
pdw = 'postgres'
|
||||
bdd = 'serenadb'
|
||||
host = '172.17.0.2'
|
||||
eng = URL.create('postgresql+psycopg2',username=usr,password=pdw,host=host,database=bdd)
|
||||
conn = create_engine(eng)
|
||||
sit = gpd.pd.read_sql_table('rnf_site',con=conn,schema='serenabase')
|
||||
sit['poly'] = (sit
|
||||
.site_poly.str[9:]
|
||||
.str.split(','))
|
||||
|
||||
sit.loc[sit.poly.notna(),'geom'] = (sit.loc[sit.poly.notna(),'poly']
|
||||
.apply(lambda x: [xx.split(' ') for xx in x if xx])
|
||||
.apply(lambda x: [[float(xxx) for xxx in xx] for xx in x ])
|
||||
.apply(lambda x: Polygon(x)))
|
||||
|
||||
return (sit
|
||||
.set_geometry('geom',crs=4326)
|
||||
.to_crs(2154)
|
||||
.dropna(subset=['geom']))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
v_synthese_invertebre = 'v_synthese_invertebre'
|
||||
v_synthese_vertebre = 'v_synthese_vertebre'
|
||||
v_synthese_flore = 'v_synthese_flore'
|
||||
|
||||
sit_rnngl = serena_rnngl_site()
|
||||
# sql_exclude_rnngl = " (rmq_localisation NOT ILIKE '%%lemps%%' OR NOT ST_INTERSECTS(geom,'SRID=2154;{}'))".format(sit_rnngl.unary_union)
|
||||
sql_exclude_rnngl = " rmq_localisation NOT ILIKE '%%grand%%lemps%%'"
|
||||
sql_rmqloc_null = " rmq_localisation IS NULL"
|
||||
|
||||
sql_inv = "SELECT * FROM saisie.%s WHERE cd_nom <> '9999081'"%v_synthese_invertebre # 9999081 : Heterocera sp.
|
||||
v_inv = gpd.read_postgis(sql_inv+" AND"+sql_exclude_rnngl,con_sicen).dropna(how='all',axis=1)
|
||||
v_inv_autre = gpd.read_postgis(sql_inv+" AND"+sql_rmqloc_null,con_sicen).dropna(how='all',axis=1)
|
||||
v_inv = gpd.pd.concat([v_inv,v_inv_autre],ignore_index=True).reset_index(drop=True)
|
||||
v_inv_rnngl = gpd.read_postgis(sql_inv+" AND"+sql_exclude_rnngl.replace('NOT ',''),con_sicen).dropna(how='all',axis=1)
|
||||
t1_inv = v_inv_rnngl.etude.str.contains('Haute Bourbre 2014')
|
||||
t2_inv = v_inv_rnngl.observateurs_v2.isin(['ZARADZKI Lise (CEN Isère)','BIRON Nicolas (CEN Isère)'])
|
||||
t3_inv = v_inv.observateurs_v2=='LUCAS Jérémie (CEN Isère)'
|
||||
t4_inv = v_inv.etude!='Echange de données - GRPLS'
|
||||
hb_inv = v_inv_rnngl[t1_inv|t2_inv].copy()
|
||||
no_stg = v_inv[t3_inv&t4_inv].copy()
|
||||
v_inv = gpd.pd.concat([v_inv,hb_inv],ignore_index=True).drop_duplicates()
|
||||
v_inv_rnngl.drop(hb_inv.index,inplace=True)
|
||||
v_inv_rnngl = gpd.pd.concat([v_inv_rnngl,no_stg],ignore_index=True).drop_duplicates()
|
||||
v_inv.drop(no_stg.index,inplace=True)
|
||||
#v_synthese_invertebre = 37915
|
||||
#v_inv.shape[0]+v_inv_rnngl.shape[0] = 37914
|
||||
#différence de 1 donnée. Correspond au taxon 9999081 dans la bdd
|
||||
|
||||
sql_ver = "SELECT * FROM saisie.%s WHERE cd_nom <> '9999056'"%v_synthese_vertebre # 9999056 : Micromammalia sp. (17 données)
|
||||
v_ver = gpd.read_postgis(sql_ver+" AND"+sql_exclude_rnngl,con_sicen).dropna(how='all',axis=1)
|
||||
v_ver_autre = gpd.read_postgis(sql_ver+" AND"+sql_rmqloc_null,con_sicen).dropna(how='all',axis=1)
|
||||
v_ver = gpd.pd.concat([v_ver,v_ver_autre],ignore_index=True).reset_index(drop=True)
|
||||
v_ver_rnngl = gpd.read_postgis(sql_ver+" AND"+sql_exclude_rnngl.replace('NOT ',''),con_sicen).dropna(how='all',axis=1)
|
||||
t1_ver = v_ver_rnngl.etude.str.contains('Haute Bourbre 2014')
|
||||
t2_ver = v_ver_rnngl.observateurs_v2.isin(['ZARADZKI Lise (CEN Isère)','BIRON Nicolas (CEN Isère)'])
|
||||
t3_ver = v_ver.observateurs_v2.isin(['LUCAS Jérémie (CEN Isère)','PRUNIER Jérôme (Personnel)', 'MAILLET Grégory (CEN Isère)','LAFON Arnaud (CEN Isère)'])
|
||||
t4_ver = v_ver.etude!='Echange de données - GRPLS'
|
||||
hb_ver = v_ver_rnngl[t1_ver|t2_ver].copy()
|
||||
sg_ver = v_ver[t3_ver&t4_ver].copy()
|
||||
v_ver = gpd.pd.concat([v_ver,hb_ver],ignore_index=True).drop_duplicates()
|
||||
v_ver_rnngl.drop(hb_ver.index,inplace=True)
|
||||
v_ver_rnngl = gpd.pd.concat([v_ver_rnngl,sg_ver],ignore_index=True).drop_duplicates()
|
||||
v_ver.drop(sg_ver.index,inplace=True)
|
||||
#v_synthese_vertebre = 102175
|
||||
#v_ver.shape[0]+v_ver_rnngl.shape[0] = 102158
|
||||
#différence de 17 donnée. Correspond au taxon 9999056 dans la bdd
|
||||
|
||||
sql_flo = 'SELECT * FROM saisie.%s'%v_synthese_flore
|
||||
v_flo = gpd.read_postgis(sql_flo+" WHERE"+sql_exclude_rnngl,con_sicen).dropna(how='all',axis=1)
|
||||
v_flo_autre = gpd.read_postgis(sql_flo+" WHERE"+sql_rmqloc_null,con_sicen).dropna(how='all',axis=1)
|
||||
v_flo = gpd.pd.concat([v_flo,v_flo_autre],ignore_index=True).reset_index(drop=True)
|
||||
v_flo_rnngl = gpd.read_postgis(sql_flo+" WHERE"+sql_exclude_rnngl.replace('NOT ',''),con_sicen).dropna(how='all',axis=1)
|
||||
t1_flo = v_flo_rnngl.etude.str.contains('Haute Bourbre 2014')
|
||||
t2_flo = v_flo_rnngl.observateurs_v2=='ZARADZKI Lise (CEN Isère)'
|
||||
t3_flo = v_flo.observateurs_v2=='LUCAS Jérémie (CEN Isère)'
|
||||
t4_flo = v_flo.etude!='Echange de données - GRPLS'
|
||||
hb_flo = v_flo_rnngl[t1_flo|t2_flo].copy()
|
||||
sg_flo = v_flo[t3_flo&t4_flo].copy()
|
||||
v_flo = gpd.pd.concat([v_flo,hb_flo],ignore_index=True).drop_duplicates()
|
||||
v_flo_rnngl.drop(hb_flo.index,inplace=True)
|
||||
idflo_drop = v_flo[v_flo.id_obs.isin(v_flo_rnngl.id_obs)].index
|
||||
v_flo.drop(idflo_drop,inplace=True)
|
||||
v_flo_rnngl = gpd.pd.concat([v_flo_rnngl,sg_flo],ignore_index=True).drop_duplicates()
|
||||
v_flo.drop(sg_flo.index,inplace=True)
|
||||
#v_synthese_flore = 113556
|
||||
#v_flo.shape[0]+v_flo_rnngl.shape[0] = 113556
|
||||
|
||||
|
||||
v_rnngl = gpd.pd.concat([v_ver_rnngl,v_flo_rnngl,v_inv_rnngl],ignore_index=True)
|
||||
|
||||
v_inv_explode = format_data(v_inv)
|
||||
V_VER = v_ver.copy()
|
||||
v_ver = V_VER.copy()
|
||||
v_ver_explode = format_data(v_ver)
|
||||
format_data(v_flo)
|
||||
|
||||
inv_uuid = (gpd.pd.read_csv(os.path.join(DIR,'INVERTEBRE/','ORB_UUID','donnees_importees_cen38.csv'),sep=';')
|
||||
.rename(columns={'entity_source_pk_value':'id_obs'}))
|
||||
v_inv_explode = v_inv_explode.merge(inv_uuid[['id_obs', 'unique_id_sinp']],how='left',on='id_obs')
|
||||
# list_inv = v_inv_explode[v_inv_explode.unique_id_sinp.notna()].etude.unique()
|
||||
# INVERTEBRE
|
||||
for etude in v_inv_explode.etude.unique():
|
||||
exp_inv = v_inv_explode[v_inv_explode.etude==etude].copy()
|
||||
exp_inv.dropna(how='all',inplace=True,axis=1)
|
||||
if 'complexe_groupe' in exp_inv.columns:
|
||||
exp_inv1 = exp_inv[exp_inv.complexe_groupe.notna()]
|
||||
exp_inv2 = exp_inv[exp_inv.complexe_groupe.isna()].dropna(how='all',axis=1)
|
||||
export(os.path.join(DIR,'INVERTEBRE','INV_'+etude+'_GRP'),exp_inv1,format='csv')
|
||||
export(os.path.join(DIR,'INVERTEBRE','INV_'+etude),exp_inv2,format='csv')
|
||||
else :
|
||||
export(os.path.join(DIR,'INVERTEBRE','INV_'+etude),exp_inv,format='csv')
|
||||
|
||||
# VERTEBRE
|
||||
for etude in v_ver_explode.etude.unique():
|
||||
exp_ver = v_ver_explode[v_ver_explode.etude==etude].copy()
|
||||
exp_ver.dropna(how='all',inplace=True,axis=1)
|
||||
if 'complexe_groupe' in exp_ver.columns:
|
||||
exp_ver1 = exp_ver[exp_ver.complexe_groupe.notna()]
|
||||
exp_ver2 = exp_ver[exp_ver.complexe_groupe.isna()].dropna(how='all',axis=1)
|
||||
export(os.path.join(DIR,'VERTEBRE','VER_'+etude+'_GRP'),exp_ver1,format='csv')
|
||||
export(os.path.join(DIR,'VERTEBRE','VER_'+etude),exp_ver2,format='csv')
|
||||
else :
|
||||
export(os.path.join(DIR,'VERTEBRE','VER_'+etude),exp_ver,format='csv')
|
||||
|
||||
# FLORE
|
||||
for etude in v_flo.etude.unique():
|
||||
exp_flo = v_flo[v_flo.etude==etude].copy()
|
||||
exp_flo.dropna(how='all',inplace=True,axis=1)
|
||||
if 'complexe_groupe' in exp_flo.columns:
|
||||
exp_flo1 = exp_flo[exp_flo.complexe_groupe.notna()]
|
||||
exp_flo2 = exp_flo[exp_flo.complexe_groupe.isna()].dropna(how='all',axis=1)
|
||||
export(os.path.join(DIR,'FLORE','FLO_'+etude+'_locNULL'+'_GRP'),exp_flo1,format='csv')
|
||||
export(os.path.join(DIR,'FLORE','FLO_'+etude+'_locNULL'),exp_flo2,format='csv')
|
||||
else :
|
||||
export(os.path.join(DIR,'FLORE','FLO_'+etude+'_locNULL'),exp_flo,format='csv')
|
||||
|
||||
|
||||
# _export(os.path.join(DIR,v_synthese_invertebre+'2'),v_inv_explode.dropna(how='all',axis=1))
|
||||
# _export(os.path.join(DIR,v_synthese_invertebre),v_inv.dropna(how='all',axis=1))
|
||||
# _export(os.path.join(DIR,v_synthese_vertebre+'2'),v_ver_explode.dropna(how='all',axis=1))
|
||||
# _export(os.path.join(DIR,v_synthese_vertebre),v_ver.dropna(how='all',axis=1))
|
||||
# _export(os.path.join(DIR,v_synthese_flore),v_flo)
|
||||
|
||||
# v_ver.etude.unique()
|
||||
# v_ver.protocole.unique()
|
||||
# v_ver.lot_donnee.unique()
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: UTF-8 -*-
|
||||
from pycen import con_sicen, update_to_sql
|
||||
import pandas as pd
|
||||
import geopandas as gpd
|
||||
from os import path
|
||||
|
||||
def get_site_serena():
|
||||
sql = '''
|
||||
SELECT s.*,
|
||||
t.*,
|
||||
c.choi_nom
|
||||
FROM serenabase.rnf_site s
|
||||
JOIN serenabase.tmp_sgll t ON t.sgll_site_id = s.site_id
|
||||
JOIN serenabase.rnf_choi c ON c.choi_id = s.site_categ_choi_id
|
||||
'''
|
||||
return pd.read_sql_query(sql,con_sicen).dropna(axis=1,how='all')
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
# Lecture des sites SIG
|
||||
PATH = '/media/cgeier/SRV/FICHIERS/OUTILS/BASES DE DONNEES/RHOMEO/2018/RhoMéO_Données'
|
||||
FILE = '38_sig_points_de_suivis_point.shp'
|
||||
sig = gpd.read_file(path.join(PATH,FILE))
|
||||
|
||||
# Récupération des sites SERENA
|
||||
site_serena = get_site_serena()
|
||||
no_coord = site_serena.sgll_lat=='999'
|
||||
is_rhomeo = site_serena.site_nom.str[-3:].str.isnumeric()
|
||||
is_rhomeo2 = site_serena.choi_nom.str.contains('rhomeo',case=False)
|
||||
# site_serena[no_coord]
|
||||
# site_serena[is_rhomeo]
|
||||
|
||||
# Isolation des sites RHOMEO
|
||||
site_rhomeo_all = site_serena[is_rhomeo2&no_coord].copy()
|
||||
site_rhomeo = site_rhomeo_all.merge(sig[['id_regio','geometry']],left_on='site_ref_sig',right_on='id_regio')\
|
||||
.set_geometry('geometry',crs=2154)
|
||||
site_rhomeo.to_crs(4326,inplace=True)
|
||||
site_rhomeo.sgll_lon = site_rhomeo.geometry.x.astype(str)
|
||||
site_rhomeo.sgll_lat = site_rhomeo.geometry.y.astype(str)
|
||||
updt_site = site_rhomeo[['sgll_site_id','sgll_lat','sgll_lon']].copy()
|
||||
|
||||
# Update data
|
||||
update_to_sql(
|
||||
df=updt_site,
|
||||
con=con_sicen,
|
||||
table_name='tmp_sgll',
|
||||
schema_name='serenabase',
|
||||
key_name='sgll_site_id'
|
||||
)
|
||||
@@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: UTF-8 -*-
|
||||
import pandas_access as mdb # pour lecture de la bd medwet
|
||||
from os import path
|
||||
from pycen import con_sicen, update_to_sql
|
||||
import pandas as pd
|
||||
|
||||
# access / postgis
|
||||
SOURCE = 'postgis'
|
||||
postgis_sch = 'serenabase'
|
||||
|
||||
DICT_RLV_COLS = {
|
||||
'relv_id':'id_etude',
|
||||
'relv_nom':'nom_etude',
|
||||
'relv_prop_libel':'proprietaire',
|
||||
'relv_1date_c':'date_debut',
|
||||
'relv_2date_c':'date_fin',
|
||||
'relv_comment':'description',
|
||||
}
|
||||
|
||||
DICT_OBS_COLS = {
|
||||
'obse_id':'id_origine',
|
||||
'obse_relv_id':'id_etude',
|
||||
}
|
||||
|
||||
|
||||
def list_access_datatable():
|
||||
|
||||
PATH_IN = '/media/cgeier/SRV/FICHIERS/OUTILS/BASES DE DONNEES/ESPECES/backup'
|
||||
BDD_IN = 'AVENIR.serena_V2.mdb'
|
||||
|
||||
LST_SCHEMA = mdb.read_schema(path.join(PATH_IN,BDD_IN))
|
||||
LST_TABLE = [ *LST_SCHEMA.keys() ]
|
||||
LST_TABLE = mdb.list_tables(path.join(PATH_IN,BDD_IN))
|
||||
TABLE_DATA= []
|
||||
|
||||
for TABLE in LST_TABLE:
|
||||
DTYPE = {x : float for x in LST_SCHEMA[TABLE].keys() if LST_SCHEMA[TABLE][x]=='Long Integer'}
|
||||
df = mdb.read_table(path.join(PATH_IN,BDD_IN),TABLE, dtype=DTYPE)
|
||||
if df.shape[0] == 0:
|
||||
print('%s IS EMPTY !'%TABLE)
|
||||
else :
|
||||
TABLE_DATA += [TABLE]
|
||||
return TABLE_DATA
|
||||
|
||||
|
||||
def list_releve(source='postgis'):
|
||||
if source=='postgis':
|
||||
|
||||
choi = pd.read_sql_table('rnf_choi',con_sicen,postgis_sch)\
|
||||
.dropna(axis=1,how='all')
|
||||
relv = pd.read_sql_table('rnf_relv',con_sicen,postgis_sch)\
|
||||
.dropna(axis=1,how='all')
|
||||
|
||||
# Correction coquille
|
||||
relv.loc[relv.relv_categ_choi_id==100800,'relv_categ_choi_id'] = 100008
|
||||
|
||||
# Jointure relevé / catégorie
|
||||
df = relv.merge(choi[['choi_id','choi_nom']],how='inner',left_on='relv_categ_choi_id',right_on='choi_id')\
|
||||
.drop(columns=['relv_categ_choi_id'])
|
||||
|
||||
# Mise en forme des dates
|
||||
lstcol_dat = df.columns[df.columns.str.contains('date')]
|
||||
for lcd in lstcol_dat:
|
||||
# df[lcd] = df[lcd].replace({'VIDE':None})
|
||||
df.loc[df[lcd]=='2000 à 2001',lcd] = '2000' if '1date' in lcd else '2001'
|
||||
df[lcd].replace({'VIDE':None},inplace=True)
|
||||
df[lcd] = pd.to_datetime(df[lcd])
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def get_serena_obs(source='postgis'):
|
||||
if source=='postgis':
|
||||
obs = pd.read_sql_table('rnf_obse',con_sicen,postgis_sch)
|
||||
|
||||
return obs
|
||||
|
||||
|
||||
|
||||
def get_sicen_obs():
|
||||
return pd.read_sql_table('saisie_observation',con_sicen,'saisie')
|
||||
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
|
||||
if SOURCE == 'access':
|
||||
lst_tab = list_access_datatable()
|
||||
elif SOURCE == 'postgis':
|
||||
lstrelv_serena = list_releve()
|
||||
obs_serena = get_serena_obs()
|
||||
obs_serena = obs_serena.merge(lstrelv_serena[['relv_id','relv_nom']],left_on='obse_relv_id',right_on='relv_id')\
|
||||
.drop(columns='relv_id')
|
||||
|
||||
# Isolation des relevés HORS RhoMéo
|
||||
is_relv = [x for x in lstrelv_serena.relv_nom.str[:3].unique() if not x.isnumeric()]
|
||||
relv_serena = lstrelv_serena[lstrelv_serena.relv_nom.str[:3].isin(is_relv)]
|
||||
|
||||
# Formatage des relevés HORS RhoMéo
|
||||
relv_2sicen = relv_serena.rename(columns=DICT_RLV_COLS).filter(DICT_RLV_COLS.values(),axis=1)
|
||||
relv_2sicen.sort_values('id_etude',inplace=True)
|
||||
relv_2sicen.reset_index(inplace=True,drop=True)
|
||||
relv_2sicen.set_index(-relv_2sicen.index,inplace=True)
|
||||
|
||||
|
||||
# # Intégration des relevés SERENA dans la table 'etude'
|
||||
# relv_2sicen.drop(columns=['id_etude','proprietaire'])\
|
||||
# .rename_axis('id_etude')\
|
||||
# .to_sql('etude',con_sicen,'md',if_exists='append')
|
||||
|
||||
|
||||
# Récupération des données SICEN2
|
||||
# id_lot = 'SERENA'
|
||||
sicen_obs = get_sicen_obs()
|
||||
is_serena = sicen_obs.id_lot==3
|
||||
is_horsetude = sicen_obs.id_etude==2
|
||||
lst_idorigine = sicen_obs[is_serena].id_origine.astype(int).tolist()
|
||||
|
||||
# isolation des données "suposées" non présentes
|
||||
obs_notinsicen = obs_serena[~obs_serena.obse_id.isin(lst_idorigine)].copy()
|
||||
# rhomeo_notinsicen = obs_notinsicen.obse_nom.str.startswith('RhoMéO',na=False)
|
||||
# obs_notinsicen[rhomeo_notinsicen]
|
||||
# test = obs_notinsicen.obse_site_id == 101335
|
||||
# obs_notinsicen[test]
|
||||
|
||||
sql1 = "SELECT * FROM {sch}.tmp_ogll WHERE ogll_obse_id IN {lst} AND ogll_lat <> '999'"
|
||||
res1 = pd.read_sql_query(sql1.format(sch=postgis_sch,lst=tuple(obs_notinsicen.obse_id)),con_sicen)
|
||||
sql2 = "SELECT * FROM {sch}.tmp_sgll WHERE sgll_site_id IN {lst} AND sgll_lat <> '999'"
|
||||
res2 = pd.read_sql_query(sql2.format(sch=postgis_sch,lst=tuple(obs_notinsicen.obse_site_id.unique())),con_sicen)
|
||||
|
||||
|
||||
miss_serena1 = obs_notinsicen.merge(
|
||||
res1.rename(columns={
|
||||
'ogll_obse_id':'obse_id',
|
||||
'ogll_lat':'lat',
|
||||
'ogll_lon':'lon'
|
||||
}),
|
||||
on='obse_id')
|
||||
miss_serena2 = obs_notinsicen.merge(
|
||||
res2.rename(columns={
|
||||
'sgll_site_id':'obse_site_id',
|
||||
'sgll_lat':'lat',
|
||||
'sgll_lon':'lon'
|
||||
}),
|
||||
on='obse_site_id')
|
||||
miss_serena = pd.concat([miss_serena1,miss_serena2])
|
||||
rhomeo_missserena = miss_serena.obse_nom.str.startswith('RhoMéO',na=False)
|
||||
miss_serena[rhomeo_missserena]
|
||||
|
||||
# Formatages des données OBS
|
||||
obs_2sicen = obs_serena[obs_serena.obse_relv_id.isin(relv_serena.relv_id)].rename(columns=DICT_OBS_COLS).filter(DICT_OBS_COLS.values(),axis=1)
|
||||
obs_2sicen['id_lot'] = 3
|
||||
obs_2sicen.id_origine = obs_2sicen.id_origine.astype(str)
|
||||
|
||||
# Correspondance id_relv (SERENA) / id_etude (SICEN2)
|
||||
DICT_ID_ETUDE = dict(zip(relv_2sicen.id_etude,relv_2sicen.index))
|
||||
obs_2sicen.id_etude.replace(DICT_ID_ETUDE,inplace=True)
|
||||
|
||||
# Mise à jour du champs id_etude de la table saisie_observation
|
||||
update_to_sql(
|
||||
obs_2sicen[obs_2sicen.id_origine.astype(int).isin(lst_idorigine)],
|
||||
con_sicen,
|
||||
'saisie_observation',
|
||||
'saisie',
|
||||
['id_lot','id_origine']
|
||||
)
|
||||
|
||||
|
||||
# identification des données restantes taguées "hors etude"
|
||||
id_origine_he = sicen_obs[is_serena&is_horsetude].id_origine.astype(int)
|
||||
is_horsetude2 = obs_serena.obse_id.isin(id_origine_he)
|
||||
res_he = obs_serena[is_horsetude2].copy()
|
||||
DICT2_OBS_COLS = {**DICT_OBS_COLS,'relv_nom':'id_waypoint'}
|
||||
obs2_2sicen = res_he.rename(columns=DICT2_OBS_COLS).filter(DICT2_OBS_COLS.values(),axis=1)
|
||||
obs2_2sicen['id_lot'] = 3
|
||||
obs2_2sicen['id_etude'] = -152
|
||||
obs2_2sicen.id_origine = obs2_2sicen.id_origine.astype(str)
|
||||
|
||||
# Mise à jour du champs id_etude de la table saisie_observation
|
||||
update_to_sql(
|
||||
obs2_2sicen,
|
||||
con_sicen,
|
||||
'saisie_observation',
|
||||
'saisie',
|
||||
['id_lot','id_origine']
|
||||
)
|
||||
|
||||
@@ -0,0 +1,464 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: UTF-8 -*-
|
||||
import pandas_access as mdb
|
||||
from pycen import con_sicen, update_to_sql
|
||||
import pandas as pd
|
||||
import geopandas as gpd
|
||||
from os import path
|
||||
|
||||
|
||||
postgis_sch = 'serenabase'
|
||||
ACCESS_PATH = '/media/cgeier/SRV/FICHIERS/OUTILS/BASES DE DONNEES/RHOMEO/2012/CEN-38_RHOMEO_BDD_2012'
|
||||
ACCESS_FILE = 'BD_Rhomeo_data.mdb'
|
||||
DICT_COLS = {
|
||||
'obse_id':'id_origine',
|
||||
}
|
||||
|
||||
|
||||
def get_columns_oftable(tab,sch,con):
|
||||
res = con.dialect.get_columns(con,tab,sch)
|
||||
return [x['name'] for x in res]
|
||||
|
||||
|
||||
def get_columns_dtypes(tab,sch,con,df:pd.DataFrame=None):
|
||||
res = con.dialect.get_columns(con,tab,sch)
|
||||
if isinstance(df,pd.DataFrame):
|
||||
return {x['name']:x['type'] for x in res if x['name'] in df.columns}
|
||||
else:
|
||||
return {x['name']:x['type'] for x in res}
|
||||
|
||||
|
||||
def match_columns(df,tab,sch,con):
|
||||
cols_saisie = get_columns_oftable(tab,sch,con)
|
||||
obs_cols = df.columns
|
||||
return obs_cols[obs_cols.isin(cols_saisie)]
|
||||
|
||||
|
||||
def lower(df):
|
||||
for c in df.columns :
|
||||
df[c] = df[c].str.lower() if df[c].apply(type).eq(str).all() else df[c]
|
||||
return df
|
||||
|
||||
|
||||
def get_odat_obse(id_obse:gpd.np.ndarray|list|pd.Series|int=None):
|
||||
sql = '''
|
||||
SELECT
|
||||
o.odat_obse_id,
|
||||
c.choi_nom,
|
||||
l.list_nom,
|
||||
o.odat_nom
|
||||
FROM {sch}.rnf_odat o
|
||||
LEFT JOIN {sch}.rnf_choi c ON c.choi_id = o.odat_choi_id
|
||||
LEFT JOIN {sch}.rnf_list l ON l.list_id = o.odat_c_list_id
|
||||
'''.format(sch=postgis_sch)
|
||||
|
||||
if isinstance(id_obse,int) :
|
||||
sql += ' WHERE odat_obse_id = %i'%id_obse
|
||||
elif isinstance(id_obse,(list,gpd.pd.Series,gpd.np.ndarray)) :
|
||||
sql += ' WHERE odat_obse_id IN {lst_id}'.format(lst_id=tuple(id_obse))
|
||||
|
||||
result = pd.read_sql_query(sql,con_sicen,index_col=['odat_obse_id','list_nom']).replace({r' \(RhoMeO\)':'',r'\?':'',r'[.]':''},regex=True)
|
||||
|
||||
df = lower(result).pivot(columns='choi_nom',values='odat_nom').reset_index(drop=False)
|
||||
|
||||
return format_comportement(df)
|
||||
|
||||
|
||||
def format_comportement(oda:pd.DataFrame):
|
||||
DICT = {
|
||||
'exuvie/émergent':'ODO_Exuvie/émergence',
|
||||
'mâles+femelles':'ODO_Mâles+Femelles',
|
||||
'autre':'ODO_Autre',
|
||||
'territoriale':'ODO_Territorial',
|
||||
'ponte':'ODO_Ponte',
|
||||
'accouplement':'ODO_Tandem',
|
||||
'tandem':'ODO_Tandem',
|
||||
'aucun':None,
|
||||
}
|
||||
return oda.replace(DICT)
|
||||
|
||||
|
||||
def format_odat(df,odat):
|
||||
DICT_COLS = {
|
||||
'comportement':'reprostatut',
|
||||
'strate':'type_effectif',
|
||||
'rmq_odat':'remarque_obs'
|
||||
}
|
||||
|
||||
|
||||
df_odat = df[['obse_id','obse_nombre','obse_nom','cd_nom','ordre']].merge(
|
||||
odat,
|
||||
left_on='obse_id',right_on='odat_obse_id'
|
||||
)
|
||||
|
||||
nb_cols = odat.columns[odat.columns.str.startswith('nb')]
|
||||
df_odat['rmq_odat'] = None
|
||||
for c in nb_cols:
|
||||
test1 = df_odat.obse_nombre!=df_odat[c]
|
||||
test2 = df_odat[c].notna()
|
||||
test3 = df_odat.rmq_odat.notna()
|
||||
test = test1&test2&test3
|
||||
tes = test1&test2&~test3
|
||||
if any(tes) or any(test):
|
||||
tt = '{rmq_o}; {c}:{data}'.format(rmq_o=df_odat[test].rmq_odat,c=c,data=df_odat[test][c])
|
||||
df_odat.loc[test,'rmq_odat'] = df_odat[test].rmq_odat+';'+c+':'+df_odat[test][c]
|
||||
df_odat.loc[tes,'rmq_odat'] = c+':'+df_odat[tes][c]
|
||||
|
||||
return df_odat.rename(columns=DICT_COLS)[['odat_obse_id',*DICT_COLS.values()]]
|
||||
|
||||
|
||||
|
||||
def get_obs_serena(insicen:bool=None):
|
||||
sql = '''
|
||||
SELECT o.*,
|
||||
s.site_nom,
|
||||
s.site_ref_sig,
|
||||
CASE WHEN t.taxo_id = 203471 THEN (SELECT cd_nom FROM inpn.taxref WHERE cd_nom = '54213' )
|
||||
WHEN t.taxo_id = 203491 THEN (SELECT cd_nom FROM inpn.taxref WHERE cd_nom = '521494' )
|
||||
WHEN tax.cd_nom IS NULL THEN ta.cd_nom::text
|
||||
ELSE tax.cd_nom
|
||||
END cd_nom,
|
||||
CASE WHEN t.taxo_id = 203471 THEN (SELECT phylum FROM inpn.taxref WHERE cd_nom = '54213' )
|
||||
WHEN t.taxo_id = 203491 THEN (SELECT phylum FROM inpn.taxref WHERE cd_nom = '521494' )
|
||||
WHEN tax.phylum IS NULL THEN ta.phylum
|
||||
ELSE tax.phylum
|
||||
END phylum,
|
||||
CASE WHEN t.taxo_id = 203471 THEN (SELECT classe FROM inpn.taxref WHERE cd_nom = '54213' )
|
||||
WHEN t.taxo_id = 203491 THEN (SELECT classe FROM inpn.taxref WHERE cd_nom = '521494' )
|
||||
WHEN tax.classe IS NULL THEN ta.classe
|
||||
ELSE tax.classe
|
||||
END classe,
|
||||
CASE WHEN t.taxo_id = 203471 THEN (SELECT ordre FROM inpn.taxref WHERE cd_nom = '54213' )
|
||||
WHEN t.taxo_id = 203491 THEN (SELECT ordre FROM inpn.taxref WHERE cd_nom = '521494' )
|
||||
WHEN tax.ordre IS NULL THEN ta.ordre
|
||||
ELSE tax.ordre
|
||||
END ordre,
|
||||
CASE WHEN t.taxo_id = 203471 THEN (SELECT famille FROM inpn.taxref WHERE cd_nom = '54213' )
|
||||
WHEN t.taxo_id = 203491 THEN (SELECT famille FROM inpn.taxref WHERE cd_nom = '521494' )
|
||||
WHEN tax.famille IS NULL THEN ta.famille
|
||||
ELSE tax.famille
|
||||
END famille,
|
||||
CASE WHEN t.taxo_id = 203471 THEN (SELECT nom_complet FROM inpn.taxref WHERE cd_nom = '54213' )
|
||||
WHEN t.taxo_id = 203491 THEN (SELECT nom_complet FROM inpn.taxref WHERE cd_nom = '521494' )
|
||||
WHEN tax.nom_complet IS NULL THEN ta.nom_complet
|
||||
ELSE tax.nom_complet
|
||||
END nom_complet,
|
||||
CASE WHEN t.taxo_id = 203471 THEN (SELECT nom_vern FROM inpn.taxref WHERE cd_nom = '54213' )
|
||||
WHEN t.taxo_id = 203491 THEN (SELECT nom_vern FROM inpn.taxref WHERE cd_nom = '521494' )
|
||||
WHEN tax.nom_vern IS NULL THEN ta.nom_vern
|
||||
ELSE tax.nom_vern
|
||||
END nom_vern,
|
||||
CASE WHEN tmpo.ogll_lon = '999' THEN tmps.sgll_lon::float
|
||||
ELSE tmpo.ogll_lon::float
|
||||
END lon,
|
||||
CASE WHEN tmpo.ogll_lat = '999' THEN tmps.sgll_lat::float
|
||||
ELSE tmpo.ogll_lat::float
|
||||
END lat
|
||||
FROM {sch}.rnf_obse o
|
||||
LEFT JOIN {sch}.rnf_relv r ON r.relv_id = o.obse_relv_id
|
||||
LEFT JOIN {sch}.rnf_site s ON s.site_id = o.obse_site_id
|
||||
LEFT JOIN {sch}.tmp_sgll tmps ON tmps.sgll_site_id = o.obse_site_id
|
||||
LEFT JOIN {sch}.tmp_ogll tmpo ON tmpo.ogll_obse_id = o.obse_id
|
||||
LEFT JOIN serenarefe.rnf_taxo t ON t.taxo_id = o.obse_taxo_id
|
||||
LEFT JOIN inpn.taxons_isere_absents_taxref ta ON ta.id_taxon = o.obse_taxo_id
|
||||
LEFT JOIN inpn.taxref tax ON t.taxo_mnhn_id = tax.cd_nom::int
|
||||
--LEFT JOIN {sch}.rnf_odat od ON od.odat_obse_id = o.obse_id
|
||||
|
||||
'''.format(sch=postgis_sch)
|
||||
|
||||
if insicen is not None :
|
||||
sql_sicen = 'SELECT DISTINCT id_origine FROM saisie.saisie_observation WHERE id_lot=3'
|
||||
sicen_obs = pd.read_sql_query(sql_sicen,con_sicen)
|
||||
if insicen is True :
|
||||
sql += 'WHERE obse_id IN {lst_id}'.format(lst_id=tuple(sicen_obs.id_origine.astype(int)))
|
||||
if insicen is False :
|
||||
sql += 'WHERE obse_id NOT IN {lst_id}'.format(lst_id=tuple(sicen_obs.id_origine.astype(int)))
|
||||
|
||||
return pd.read_sql_query(sql,con_sicen)\
|
||||
.dropna(axis=1,how='all')
|
||||
|
||||
|
||||
def list_releve(id_relv=None,source='postgis'):
|
||||
if source=='postgis':
|
||||
|
||||
df_choi = pd.read_sql_table('rnf_choi',con_sicen,postgis_sch)\
|
||||
.dropna(axis=1,how='all')
|
||||
df_relv = pd.read_sql_table('rnf_relv',con_sicen,postgis_sch)\
|
||||
.dropna(axis=1,how='all')
|
||||
|
||||
# Correction coquille
|
||||
df_relv.loc[df_relv.relv_categ_choi_id==100800,'relv_categ_choi_id'] = 100008
|
||||
|
||||
# Jointure relevé / catégorie
|
||||
df = df_relv.merge(df_choi[['choi_id','choi_nom']],how='inner',left_on='relv_categ_choi_id',right_on='choi_id')\
|
||||
.drop(columns=['relv_categ_choi_id'])
|
||||
|
||||
# Mise en forme des dates
|
||||
lstcol_dat = df.columns[df.columns.str.contains('date')]
|
||||
for lcd in lstcol_dat:
|
||||
# df[lcd] = df[lcd].replace({'VIDE':None})
|
||||
df.loc[df[lcd]=='2000 à 2001',lcd] = '2000' if '1date' in lcd else '2001'
|
||||
df[lcd].replace({'VIDE':None},inplace=True)
|
||||
df[lcd] = pd.to_datetime(df[lcd])
|
||||
|
||||
if id_relv is not None:
|
||||
if isinstance(id_relv,int) : filtre = df.relv_id == id_relv
|
||||
elif isinstance(id_relv,(list,gpd.pd.Series,gpd.np.ndarray)) : filtre = df.relv_id.isin([*id_relv])
|
||||
return df[filtre]
|
||||
else:
|
||||
return df
|
||||
|
||||
|
||||
def get_serena_obs(srce_id=None,source:str='access'):
|
||||
DICT_STR = {
|
||||
'avenir':'cen isère',
|
||||
'be':'personnel'
|
||||
}
|
||||
if source == 'postgis':
|
||||
sql = 'SELECT user_id,user_srce_id FROM {sch}.rnf_user'.format(sch=postgis_sch)
|
||||
if srce_id is not None:
|
||||
if isinstance(srce_id,int) :
|
||||
sql += ' WHERE user_srce_id = %i'%srce_id
|
||||
elif isinstance(srce_id,(list,gpd.pd.Series,gpd.np.ndarray)) :
|
||||
sql += ' WHERE user_srce_id IN {lst_id}'.format(lst_id=tuple(srce_id))
|
||||
return lower(pd.read_sql_query(sql,con_sicen))
|
||||
|
||||
elif source == 'access':
|
||||
df = mdb.read_table(path.join(ACCESS_PATH,ACCESS_FILE),'rev_obs')
|
||||
if srce_id is not None:
|
||||
if isinstance(srce_id,int) :
|
||||
df = df[df.rev_observateur==srce_id]
|
||||
elif isinstance(srce_id,(list,gpd.pd.Series,gpd.np.ndarray)) :
|
||||
df = df[df.rev_observateur.isin([*srce_id])]
|
||||
df[['nom','prenom']] = df.nom_observateur.str.split(' ',1,expand=True)
|
||||
|
||||
return lower(df).replace(DICT_STR)
|
||||
|
||||
|
||||
def get_sicen_obs(personne_id=None):
|
||||
sql = 'SELECT id_personne,prenom,nom,id_structure,nom_structure FROM md.personne JOIN md.structure USING (id_structure)'.format(sch=postgis_sch)
|
||||
if personne_id is not None:
|
||||
if isinstance(personne_id,int) :
|
||||
sql += ' WHERE personne_id = %i'%personne_id
|
||||
elif isinstance(personne_id,str) :
|
||||
sql += ' WHERE personne_id = %i'%int(personne_id)
|
||||
elif isinstance(personne_id,(list,gpd.pd.Series,gpd.np.ndarray)) :
|
||||
sql += ' WHERE personne_id IN {lst_id}'.format(lst_id=tuple(personne_id))
|
||||
return lower(pd.read_sql_query(sql,con_sicen))
|
||||
|
||||
def get_sicen_etude():
|
||||
sql = 'SELECT * FROM md.etude'
|
||||
return pd.read_sql_query(sql, con_sicen)
|
||||
|
||||
|
||||
def get_serena_choi(choi_id:gpd.np.ndarray|list|pd.Series|int=None):
|
||||
''''''
|
||||
sql = 'SELECT choi_id,choi_nom FROM {sch}.rnf_choi'.format(sch=postgis_sch)
|
||||
|
||||
if isinstance(choi_id,int) :
|
||||
sql += ' WHERE choi_id = %i'%choi_id
|
||||
elif isinstance(choi_id,(list,gpd.pd.Series,gpd.np.ndarray)) :
|
||||
sql += ' WHERE choi_id IN {lst_id}'.format(lst_id=tuple(choi_id))
|
||||
|
||||
return lower(pd.read_sql_query(sql,con_sicen))
|
||||
|
||||
|
||||
def get_serena_site(site_id:gpd.np.ndarray|list|pd.Series|int=None):
|
||||
sql = 'SELECT site_id,site_nom FROM {sch}.rnf_site'.format(sch=postgis_sch)
|
||||
|
||||
if isinstance(site_id,int) :
|
||||
sql += ' WHERE site_id = %i'%site_id
|
||||
elif isinstance(site_id,(list,gpd.pd.Series,gpd.np.ndarray)) :
|
||||
sql += ' WHERE site_id IN {lst_id}'.format(lst_id=tuple(site_id))
|
||||
return pd.read_sql_query(sql,con_sicen)
|
||||
|
||||
|
||||
def get_sicen_pcol():
|
||||
sql = '''SELECT * FROM "md"."protocole";'''
|
||||
return lower(pd.read_sql_query(sql,con_sicen))
|
||||
|
||||
|
||||
def _crsp(left:pd.DataFrame,right:pd.DataFrame,lefton,righton,idleft,idright):
|
||||
df = left.merge(right,left_on=lefton,right_on=righton)
|
||||
return dict(zip(df[idleft],df[idright]))
|
||||
|
||||
|
||||
def crsp_relv(relv:pd.Series):
|
||||
seren_relv = lower(list_releve(relv.unique()))
|
||||
sicen_relv = lower(get_sicen_etude())
|
||||
DICT_RELV_ID = _crsp(left=seren_relv,right=sicen_relv,lefton='relv_nom',righton='nom_etude',idleft='relv_id',idright='id_etude')
|
||||
return relv.replace(DICT_RELV_ID)
|
||||
|
||||
|
||||
def crsp_obs(obs:pd.Series):
|
||||
obs_seren = get_serena_obs(obs.unique())
|
||||
obs_sicen = get_sicen_obs()
|
||||
DICT_OBS_ID = _crsp(left=obs_seren,right=obs_sicen,lefton=['nom','structure_obs'],righton=['nom','nom_structure'],idleft='rev_observateur',idright='id_personne')
|
||||
return obs.replace(DICT_OBS_ID)
|
||||
|
||||
|
||||
def crsp_pcol(pcol:pd.Series):
|
||||
pcol_seren = get_serena_choi(pcol.unique())
|
||||
pcol_sicen = get_sicen_pcol()
|
||||
DICT_PCOL_ID = _crsp(left=pcol_seren,right=pcol_sicen,lefton='choi_nom',righton='libelle',idleft='choi_id',idright='id_protocole')
|
||||
return pcol.replace(DICT_PCOL_ID)
|
||||
|
||||
|
||||
def crsp_valid(valid:pd.Series):
|
||||
pcol_seren = get_serena_choi(valid.unique())
|
||||
pcol_sicen = get_sicen_pcol()
|
||||
DICT_PCOL_ID = _crsp(left=pcol_seren,right=pcol_sicen,lefton='choi_nom',righton='libelle',idleft='choi_id',idright='id_protocole')
|
||||
return valid.replace(DICT_PCOL_ID)
|
||||
|
||||
|
||||
def crsp_abond(abond:pd.Series):
|
||||
ab_seren = get_serena_choi(abond.dropna().unique())
|
||||
DICT_ABOND_ID = dict(zip(ab_seren.choi_id,ab_seren.choi_nom))
|
||||
return abond.replace(DICT_ABOND_ID)
|
||||
|
||||
|
||||
def crsp_site(site:pd.Series):
|
||||
sit_seren = get_serena_site(site.dropna().unique())
|
||||
DICT_SITE_ID = dict(zip(sit_seren.site_id,sit_seren.site_nom))
|
||||
return site.replace(DICT_SITE_ID)
|
||||
|
||||
|
||||
def get_structure_id(id_pers:gpd.np.ndarray|list|pd.Series|int=None):
|
||||
sql = 'SELECT id_personne, id_structure FROM md.personne'
|
||||
if isinstance(id_pers,int) :
|
||||
sql += ' WHERE id_personne = %i'%id_pers
|
||||
elif isinstance(id_pers,(list,gpd.pd.Series,gpd.np.ndarray)) :
|
||||
sql += ' WHERE id_personne IN {lst_id}'.format(lst_id=tuple(id_pers))
|
||||
return pd.read_sql_query(sql,con_sicen)
|
||||
|
||||
|
||||
def format_effectif(DF):
|
||||
df = DF.copy()
|
||||
isnum = df.effectif_min.str.isnumeric()
|
||||
df.loc[isnum,'effectif'] = df[isnum].effectif_min.copy()
|
||||
df.loc[isnum,'effectif_min'] = None
|
||||
df.loc[~isnum,'effectif_textuel'] = df[~isnum].effectif_min.copy()
|
||||
df.loc[~isnum,'effectif_min'] = df[~isnum].effectif_textuel.str.split(r'[-+]',expand=True)[0]
|
||||
df.loc[~isnum,'effectif_max'] = df[~isnum].effectif_textuel.str.split(r'[-+]',expand=True)[1]
|
||||
return df.replace({'':None})
|
||||
|
||||
def to_sicen(DF):
|
||||
|
||||
CRSP_COLUMNS = {
|
||||
'obse_id':'id_origine',
|
||||
'obse_relv_id':'id_etude',
|
||||
'obse_obsv_id':'observateur',
|
||||
'obse_detm_id':'validateur',
|
||||
'obse_date':'date_obs',
|
||||
# 'obse_site_id':'localisation',
|
||||
'obse_pcole_choi_id':'id_protocole',
|
||||
'obse_validat_choi_id':'statut_validation',
|
||||
'obse_confid_choi_id':'diffusable',
|
||||
'obse_abond_choi_id':'effectif_textuel',
|
||||
'obse_nombre':'effectif_min',
|
||||
'site_nom':'localisation',
|
||||
'site_ref_sig':'id_waypoint',
|
||||
'lat':'latitude',
|
||||
'lon':'longitude',
|
||||
}
|
||||
|
||||
df = DF.copy()
|
||||
if df.geometry.name != 'geometrie' :
|
||||
df.rename_geometry('geometrie',inplace=True)
|
||||
|
||||
df.obse_relv_id = crsp_relv(df.obse_relv_id)
|
||||
df.obse_obsv_id = crsp_obs(df.obse_obsv_id)
|
||||
df.obse_detm_id = crsp_obs(df.obse_detm_id).astype(int)
|
||||
df.obse_date = pd.to_datetime(df.obse_date)
|
||||
# df.obse_site_id = crsp_site(df.obse_site_id)
|
||||
df.obse_pcole_choi_id = crsp_pcol(df.obse_pcole_choi_id)
|
||||
df.loc[df.obse_validat_choi_id==100373,'obse_validat_choi_id'] = 'validée'
|
||||
df.loc[df.obse_confid_choi_id==100473,'obse_confid_choi_id'] = True
|
||||
df.obse_abond_choi_id = crsp_abond(df.obse_abond_choi_id)
|
||||
df.rename(columns=CRSP_COLUMNS,inplace=True)
|
||||
|
||||
# Tag du lot SERENA
|
||||
df['id_lot'] = 3
|
||||
|
||||
# Jointure des id_structure
|
||||
struct = get_structure_id(df.observateur.unique())
|
||||
df = df.merge(struct,how='left',left_on='observateur', right_on='id_personne')\
|
||||
.rename(columns={'id_structure':'structure'})
|
||||
cols = match_columns(df,'saisie_observation','saisie',con_sicen)
|
||||
|
||||
|
||||
# Formatage des effectifs
|
||||
return format_effectif(df[cols])
|
||||
|
||||
|
||||
def drop_cdnom_missing(df):
|
||||
isna = df.cd_nom.isna()
|
||||
idx_na = df[isna].index
|
||||
return df.drop(idx_na)
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
obs_serena = get_obs_serena(insicen=True)
|
||||
isna = obs_serena.cd_nom.isna()
|
||||
|
||||
# Recréation des géométries
|
||||
gdf_obs = obs_serena.set_geometry(gpd.points_from_xy(obs_serena.lon,obs_serena.lat))
|
||||
gdf_obs.set_crs(4326,inplace=True)
|
||||
gdf_obs.to_crs(2154,inplace=True)
|
||||
|
||||
# Identification des données RhoMeo
|
||||
is_rhomeo = gdf_obs.obse_nom.str.contains('rhoméo',na=False,case=False)
|
||||
gdf_obs.loc[is_rhomeo,['obse_id','obse_nom','obse_date','cd_nom']]
|
||||
obs_rhomeo = gdf_obs[is_rhomeo]\
|
||||
.dropna(axis=1,how='all')\
|
||||
.drop(columns=['obse_habi_id'])
|
||||
rhoisna = obs_rhomeo.cd_nom.isna()
|
||||
obs_rhomeo = drop_cdnom_missing(obs_rhomeo)
|
||||
|
||||
odat = get_odat_obse(obs_rhomeo.obse_id)
|
||||
|
||||
OBS_RHOMEO = obs_rhomeo.merge(
|
||||
format_odat(obs_rhomeo,odat),
|
||||
left_on='obse_id',right_on='odat_obse_id',how='left'
|
||||
).drop(columns='odat_obse_id')
|
||||
|
||||
OBS_RHOMEO.to_file('/home/colas/Documents/9_PROJETS/4_SICEN/RECOVERY/rhomeo_data_notin_sicen.gpkg')
|
||||
|
||||
RES_OBS = to_sicen(OBS_RHOMEO)
|
||||
RES_OBS.to_postgis(
|
||||
'saisie_observation',
|
||||
con_sicen,
|
||||
'saisie',
|
||||
if_exists='append',
|
||||
index=False,
|
||||
dtype=get_columns_dtypes(
|
||||
'saisie_observation',
|
||||
'saisie',
|
||||
con_sicen,
|
||||
RES_OBS
|
||||
)
|
||||
)
|
||||
# .to_file('/home/colas/Documents/9_PROJETS/4_SICEN/RECOVERY/rhomeo_data_notin_sicen.gpkg')
|
||||
|
||||
|
||||
update_to_sql(
|
||||
RES_OBS[['id_lot','id_origine','effectif','effectif_min']],
|
||||
con_sicen,
|
||||
'saisie_observation',
|
||||
'saisie',
|
||||
['id_lot','id_origine'],
|
||||
dtype=get_columns_dtypes(
|
||||
'saisie_observation',
|
||||
'saisie',
|
||||
con_sicen,
|
||||
RES_OBS[['id_lot','id_origine','effectif','effectif_min']]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# Récupération des relevés
|
||||
id_relvrho = gdf_obs[is_rhomeo].obse_relv_id.unique()
|
||||
relv = list_releve(id_relv=id_relvrho)
|
||||
relv_rh = relv.relv_nom
|
||||
@@ -0,0 +1,107 @@
|
||||
from pycen import con_gn,con_sicen, update_to_sql
|
||||
import pandas as pd
|
||||
|
||||
|
||||
############################
|
||||
####### Araignée sp. #######
|
||||
######## Fungi sp. #########
|
||||
############################
|
||||
dict_tax = {
|
||||
'Pachygnatha sp':195648,
|
||||
'Clubiona sp':190950,
|
||||
'Enoplognatha sp':192179,
|
||||
'Tetragnatha sp':198316,
|
||||
'Linyphiidae sp':184366,
|
||||
'Episinus sp':192245,
|
||||
'Evarcha sp':192480,
|
||||
'Thanatus sp':198365,
|
||||
'Xysticus sp':199101,
|
||||
'Araneidae sp':184354,
|
||||
'Pardosa cf saltans':457342, # doute
|
||||
'Zora sp':199158,
|
||||
'Piratula sp':806893,
|
||||
'Pardosa sp':195830,
|
||||
'Thomisidae sp':184390,
|
||||
'Gnaphosidae sp':184362,
|
||||
'Araniella sp':189390,
|
||||
'Pirata sp':196303,
|
||||
'Trochosa sp':198691,
|
||||
'Zelotes sp':199129,
|
||||
'Agelena sp':188876,
|
||||
'Ozyptila sp':202569,
|
||||
'Metellina sp':202546,
|
||||
'Hyptiotes cf paradoxus':1058, # doute
|
||||
'Phrurolithus sp':196174,
|
||||
'Tegenaria sp':198254,
|
||||
'Singa sp':197676,
|
||||
'Heliophanus sp':193148,
|
||||
'Zodarion sp':199147,
|
||||
'Tmarus sp':198482,
|
||||
'Dysdera sp':192003,
|
||||
'Liocranidae sp':184367,
|
||||
'Micaria sp':194695,
|
||||
'Salticidae sp':184381,
|
||||
'Agroeca sp':188904,
|
||||
'Lycosidae sp':184368,
|
||||
'Zora cf spinimana':1753, # doute
|
||||
'Marpissa sp':194474,
|
||||
'Alopecosa sp':188992,
|
||||
# 'Amphinema diadema':, # Contact fournisseur données : RIVOIRE Bernard
|
||||
'Hyphodermella rosae':905273,
|
||||
'Cartilosoma rene-hentic':905220,
|
||||
# 'Phanerochaete livescens':, # Appeler CBNA / RIVOIRE Bernard
|
||||
# 'Thanatephorus brevisporus':, # Contact fournisseur données : RIVOIRE Bernard
|
||||
'Phlebia acerina':970280 }
|
||||
|
||||
|
||||
sql = """
|
||||
select id_obs,remarque_obs FROM saisie.saisie_observation WHERE cd_nom in ('9999077','9999078')
|
||||
and remarque_obs not ilike '%%Amphinema diadema%%'
|
||||
and remarque_obs not ilike '%%Phanerochaete livescens%%'
|
||||
and remarque_obs not ilike '%%Thanatephorus brevisporus%%'
|
||||
"""
|
||||
df = pd.read_sql(sql, con_sicen)
|
||||
|
||||
df['cd_nom'] = (df['remarque_obs']
|
||||
.str.split('|',n=1)
|
||||
.str[0]
|
||||
.str.replace('Nom taxon saisi :','')
|
||||
.str.strip()
|
||||
.replace(dict_tax)
|
||||
)
|
||||
|
||||
sql = 'SELECT cd_nom,regne,nom_vern,phylum,famille,classe,ordre,nom_complet,nom_valide FROM taxonomie.taxref WHERE cd_nom IN {}'
|
||||
tax = pd.read_sql(sql.format(tuple(df['cd_nom'].unique())), con_gn)
|
||||
|
||||
df = (df.merge(tax,how='left',on='cd_nom')
|
||||
.drop(columns=['remarque_obs'])
|
||||
.astype({'cd_nom':str}))
|
||||
|
||||
update_to_sql(
|
||||
df,
|
||||
con_sicen,
|
||||
'saisie_observation',
|
||||
'saisie','id_obs')
|
||||
|
||||
|
||||
############################
|
||||
######## Myotis sp. ########
|
||||
############################
|
||||
|
||||
dict_tax = {'9999058':195005}
|
||||
sql = """
|
||||
select id_obs,cd_nom FROM saisie.saisie_observation WHERE cd_nom = '9999058'
|
||||
"""
|
||||
df = pd.read_sql(sql, con_sicen)
|
||||
df.replace({'cd_nom':dict_tax},inplace=True)
|
||||
|
||||
sql = 'SELECT cd_nom,regne,nom_vern,phylum,famille,classe,ordre,nom_complet,nom_valide FROM taxonomie.taxref WHERE cd_nom = %i'
|
||||
tax = pd.read_sql(sql%df['cd_nom'].unique()[0], con_gn)
|
||||
|
||||
df = (df.merge(tax,how='left',on='cd_nom')
|
||||
.astype({'cd_nom':str}))
|
||||
update_to_sql(
|
||||
df,
|
||||
con_sicen,
|
||||
'saisie_observation',
|
||||
'saisie','id_obs')
|
||||
@@ -0,0 +1,39 @@
|
||||
import pycen
|
||||
import geopandas as gpd
|
||||
|
||||
sql = 'SELECT * FROM saisie.saisie_observation'
|
||||
df = gpd.read_postgis(sql,pycen.con_sicen,geom_col="geometrie")
|
||||
|
||||
sql = 'SELECT * FROM saisie.v_synthese_invertebre'
|
||||
v_inv = gpd.read_postgis(sql,pycen.con_sicen,geom_col="geom")
|
||||
v_inv.date_obs = gpd.pd.to_datetime(v_inv.date_obs)
|
||||
|
||||
v_inv2021 = v_inv[v_inv.date_obs > '2021-01-01']
|
||||
|
||||
sql = 'SELECT * FROM saisie.v_synthese_vertebre'
|
||||
v_vert = gpd.read_postgis(sql,pycen.con_sicen,geom_col="geom")
|
||||
v_vert.date_obs = gpd.pd.to_datetime(v_vert.date_obs)
|
||||
chiro = v_vert[v_vert.ordre=='Chiroptera']
|
||||
|
||||
dict_obs_pu = [
|
||||
'{"JUTON Mathieu"}', '{"VEILLET Bruno"}',
|
||||
'{"SUCHET Patrick","VEILLET Bruno"}',
|
||||
'{"SUCHET Patrick"}',
|
||||
'{"BIRON Nicolas"}',
|
||||
'{"PASQUIER Guillaume"}',
|
||||
'{"LUCAS Jérémie"}',
|
||||
'{"JUTON Mathieu","PASQUIER Guillaume"}',
|
||||
'{"BIRON Nicolas","BONHOMME Baptiste","GROSSI Jean-Luc","JUTON Mathieu"}',
|
||||
'{"DELOCHE Denis","BEGUIN Lucile","JUTON Mathieu"}',
|
||||
'{"BERENGER Myrtille"}','{"VINCENT Stéphane"}'
|
||||
]
|
||||
chiro[(
|
||||
(chiro.diffusable=="oui") |
|
||||
(chiro.observateurs.isin(dict_obs_pu)) |
|
||||
chiro.rmq_localisation.str.contains('Camp de Chambaran',case=False))]
|
||||
|
||||
# IDEM
|
||||
# chiro[(
|
||||
# (chiro.diffusable=="oui") |
|
||||
# (chiro.observateurs.isin(dict_obs_pu)) |
|
||||
# (chiro.rmq_localisation=='Camp de Chambaran'))]
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-.
|
||||
|
||||
from sqlalchemy import create_engine #, text
|
||||
from sqlalchemy.engine import URL
|
||||
import geopandas as gpd
|
||||
from pycen.zh import zh
|
||||
zh = zh()
|
||||
from pycen import con_sicen as con
|
||||
|
||||
file = '/home/colas/Documents/9_PROJETS/1_ZH/SIMBY_zonage_Belledonne.txt'
|
||||
|
||||
|
||||
if file :
|
||||
lst_site = gpd.pd.read_csv(file)
|
||||
lst_site = [*lst_site.site]
|
||||
|
||||
sql = """
|
||||
SELECT * FROM zones_humides.v_zoneshumides
|
||||
"""
|
||||
sql = sql + " WHERE site_code IN ('%s')" % "','".join(lst_site)
|
||||
VZH = gpd.read_postgis(sql,zh.con,geom_col='geom')
|
||||
|
||||
sql = """
|
||||
SELECT * FROM saisie.saisie_observation s
|
||||
WHERE ST_Intersects(s.geometrie,'SRID=2154;%s')
|
||||
""" % VZH.unary_union
|
||||
|
||||
saisie = gpd.read_postgis(sql,con,geom_col='geometrie')
|
||||
|
||||
tmp = gpd.sjoin(VZH[['site_code','geom']], saisie)
|
||||
ss = tmp[['site_code',
|
||||
'phylum','regne','classe','ordre','famille',
|
||||
'nom_valide','nom_vern','nom_complet','cd_nom']].copy()
|
||||
ss.drop_duplicates(inplace=True)
|
||||
ss.nom_valide = ss.nom_valide.replace([' \(#ref\)',],[''],regex=True)
|
||||
ss.nom_complet = ss.nom_complet.replace([' \(#ref\)',],[''],regex=True)
|
||||
ss.sort_values('nom_valide', inplace=True)
|
||||
sp_enjeux = [
|
||||
'Carex brunnescens (Pers.) Poir., 1813',
|
||||
'Coenagrion mercuriale (Charpentier, 1840)',
|
||||
'Drosera rotundifolia L., 1753',
|
||||
'Juncus squarrosus L., 1753',
|
||||
'Lycopodium annotinum L., 1753',
|
||||
'Lycopodium clavatum L., 1753',
|
||||
'Neottia cordata (L.) Rich., 1817',
|
||||
'Pinguicula grandiflora subsp. rosea (Mutel) Casper, 1962',
|
||||
# 'Pinguicula grandiflora Lam., 1789',
|
||||
'Pinguicula grandiflora subsp. grandiflora Lam., 1789',
|
||||
]
|
||||
ss.nom_valide.unique()
|
||||
|
||||
ss[ss.nom_valide.isin(sp_enjeux)].to_excel('/home/colas/Documents/tmp/espèced_enjeux_belledonne.xlsx')
|
||||
ss[ss.nom_complet.isin(sp_enjeux)].to_file('/home/colas/Documents/tmp/espèced_enjeux_belledonne.xlsx')
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding:UTF-8 -*-
|
||||
|
||||
|
||||
from sqlalchemy import create_engine #, text
|
||||
from sqlalchemy.engine import URL
|
||||
import geopandas as gpd
|
||||
import pandas as pd
|
||||
from pycen import con_sicen as con, con_bdcen as con38
|
||||
|
||||
|
||||
sql = 'SELECT * FROM sites.c_sites_zonages'
|
||||
site = gpd.read_postgis(sql,con38,geom_col='geom')
|
||||
site.sort_values(['code_site','surface_ha'], inplace=True)
|
||||
site.reset_index(inplace=True,drop=True)
|
||||
site.drop_duplicates(subset=['code_site'],keep='last', inplace=True)
|
||||
|
||||
|
||||
cod = 'ARSE'
|
||||
for cod in site.code_site:
|
||||
print(cod)
|
||||
sql = """
|
||||
SELECT * FROM saisie.vm_synthese_observations
|
||||
WHERE ST_Intersects(geom,'SRID=2154;%s')
|
||||
""" % site[site.code_site==cod].geom.values[0]
|
||||
obs = gpd.read_postgis(sql,con,geom_col='geom')
|
||||
|
||||
if obs.empty:
|
||||
continue
|
||||
|
||||
sql = """
|
||||
SELECT operation op, date_operation date_op, utilisateur user_op, id_obs FROM saisie.suivi_saisie_observation
|
||||
WHERE operation = 'INSERT' AND id_obs IN (%s)
|
||||
""" % ",".join(obs.id_obs.astype(str))
|
||||
ins_bos = pd.read_sql_query(
|
||||
sql,con
|
||||
)
|
||||
if ins_bos.empty: pass
|
||||
else:
|
||||
ins_bos['heure_op'] = ins_bos.date_op.dt.time
|
||||
ins_bos.date_op = ins_bos.date_op.dt.date
|
||||
ins_bos.date_op = ins_bos.date_op.astype(str)
|
||||
ins_bos.heure_op = ins_bos.heure_op.astype(str)
|
||||
|
||||
obs = pd.merge(ins_bos,obs, on=['id_obs'], how='right')
|
||||
obs.structures = [' & '.join(i) for i in obs.structures]
|
||||
obs.observateurs = [' & '.join(i) for i in obs.observateurs]
|
||||
obs.date_obs = obs.date_obs.astype(str)
|
||||
obs.date_debut_obs = obs.date_debut_obs.astype(str)
|
||||
obs.date_fin_obs = obs.date_fin_obs.astype(str)
|
||||
obs.heure_obs = obs.heure_obs.astype(str)
|
||||
obs.set_index('id_obs', inplace=True)
|
||||
|
||||
output = '/home/colas/Documents/tmp/obsbysite/%s.xlsx' % cod
|
||||
with pd.ExcelWriter(output) as writer:
|
||||
obs[obs.structures=='CEN Isère'].to_excel(writer,sheet_name='CEN Isère', index=True)
|
||||
obs[obs.structures!='CEN Isère'].to_excel(writer,sheet_name='Partenaire', index=True)
|
||||
writer.save()
|
||||
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-.
|
||||
|
||||
from sqlalchemy import create_engine #, text
|
||||
from sqlalchemy.engine import URL
|
||||
from sqlalchemy.types import Date,Time,String
|
||||
from shapely.geometry import Point
|
||||
import pandas as pd
|
||||
import datetime
|
||||
from pycen import con_bdcen as con
|
||||
|
||||
file = '/home/colas/Documents/9_PROJETS/4_SICEN/IMPORT/Tableau Saisie_FauneFlore_AG_30_03_22.xlsx'
|
||||
# file = '/home/colas/Documents/9_PROJETS/4_SICEN/LPO/cdnom_idvisionature.csv'
|
||||
to_table = 'import_data_cen38'
|
||||
# to_table = 'taxons_fauneisere_biolovision_taxref'
|
||||
|
||||
|
||||
df = pd.read_excel(
|
||||
file,
|
||||
skiprows=[1,2,3,4]
|
||||
)
|
||||
if 'nom de champ' in df.columns:
|
||||
del df['nom de champ']
|
||||
# df = pd.read_csv(file,sep=',')
|
||||
df.columns = df.columns.str.strip().str.lower()
|
||||
df.dropna(subset=['date_obs'],inplace=True)
|
||||
df.drop_duplicates(inplace=True)
|
||||
|
||||
|
||||
date_default = datetime.datetime(1899, 12, 30, 0, 0)
|
||||
# data_import.loc[data_import['time_start'] == date_default,'time_start'] = datetime.time(0,0)
|
||||
# data_import.loc[data_import['time_stop'] == date_default,'time_stop'] = datetime.time(0,0)
|
||||
# data_import.loc[data_import['time_start'] == '1899-12-30 00:00:00','time_start'] = '00:00:00'
|
||||
# data_import.loc[data_import['time_stop'] == '1899-12-30 00:00:00','time_stop'] = '00:00:00'
|
||||
# df.loc[df['horaire'] == date_default,'horaire'] = datetime.time(0,0)
|
||||
df.date_obs = df.date_obs.dt.date
|
||||
|
||||
if to_table == 'import_data_cen38':
|
||||
dtype = {
|
||||
'date_obs':Date,
|
||||
'date_debut_obs':Date,
|
||||
'date_fin_obs':Date,
|
||||
'heure':Time,
|
||||
'strate':String,
|
||||
'phenologie':String,
|
||||
'cd_nom':String,
|
||||
'sexe':String,
|
||||
'precision':String}
|
||||
else : dtype = None
|
||||
|
||||
|
||||
df.to_sql(
|
||||
name=to_table,
|
||||
con = con,
|
||||
schema='import',
|
||||
index=False,
|
||||
if_exists='replace',
|
||||
method='multi',
|
||||
dtype=dtype
|
||||
)
|
||||
|
||||
sql_grant = """
|
||||
GRANT ALL ON TABLE import.%s TO grp_admin
|
||||
""" % to_table
|
||||
|
||||
with con.begin() as cnx:
|
||||
cnx.execute(sql_grant)
|
||||
@@ -0,0 +1,319 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-.
|
||||
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.engine import URL
|
||||
from datetime import datetime as dt
|
||||
import pandas as pd
|
||||
import geopandas as gpd
|
||||
from pycen import con_sicen as con
|
||||
|
||||
nom_numerateur = 'GEIER'
|
||||
pnom_numerator = 'Colas'
|
||||
today = dt.now().date().isoformat()
|
||||
|
||||
nom_numerateur = nom_numerateur.upper()
|
||||
pnom_numerator = pnom_numerator[0].upper() + pnom_numerator[1:]
|
||||
query_get_numerator = """
|
||||
SELECT * FROM md.personne
|
||||
WHERE nom = '%s'
|
||||
AND prenom = '%s'
|
||||
AND id_structure = 1
|
||||
""" % (nom_numerateur,pnom_numerator)
|
||||
numerateur = pd.read_sql_query(
|
||||
sql = query_get_numerator,
|
||||
con = con
|
||||
)
|
||||
|
||||
|
||||
select_import = """
|
||||
SELECT * FROM import.import_data_cen38;
|
||||
"""
|
||||
data_import = pd.read_sql_query(
|
||||
sql = select_import,
|
||||
con = con)
|
||||
|
||||
|
||||
check_datageom = """
|
||||
DROP VIEW IF EXISTS import.test_geom_prestation;
|
||||
CREATE OR REPLACE VIEW import.test_geom_prestation as (
|
||||
SELECT ROW_NUMBER() OVER (ORDER BY date_obs ASC) AS gid, localisation,
|
||||
CASE WHEN latitude::numeric > 6000000
|
||||
THEN ST_GeomFromText('POINT('||longitude::double precision||' '||latitude::double precision||')',2154) -- coordonnées en lambert93
|
||||
WHEN latitude::numeric > 50 AND latitude::numeric < 3000000
|
||||
THEN ST_transform(ST_GeomFromText('POINT('||longitude::double precision||' '||latitude::double precision||')',27572),2154) -- coordonnées en Lambert II étendu
|
||||
--WHEN latitude is null THEN st_pointonsurface(b.geom) -- pas de coordonnées XY on récupère la geom du centroide du site CenRA si elle existe et si une partie du nom du site est renseignée dans la colonne LOCALISATION
|
||||
ELSE ST_transform(ST_GeomFromText('POINT('||longitude::double precision||' '||latitude::double precision||')',4326),2154) -- coordonnées en WGS84
|
||||
END as geometrie
|
||||
FROM import.import_data_cen38
|
||||
--LEFT JOIN import.sites_serena_cren_mu_2015 b ON lower(b.nom_site) like '%'||lower(trim(localisation,'s'))||'%' -- on joint la table des sites CenRA pour récupérer éventuellement le centroide d'un site si pas de XY
|
||||
WHERE cd_nom is not null and (date_obs is not null or date_debut_obs is not null)) ; -- pour ignorer les lignes vides qui traineraient dans le csv
|
||||
SELECT * FROM import.test_geom_prestation;
|
||||
"""
|
||||
df_check_datageom = gpd.read_postgis(
|
||||
sql = text(check_datageom),
|
||||
con = con,
|
||||
geom_col='geometrie')
|
||||
if df_check_datageom.geometry.is_valid.all() \
|
||||
and df_check_datageom.crs.srs == 'epsg:2154':
|
||||
drop_datageom = """DROP VIEW IF EXISTS import.test_geom_prestation;"""
|
||||
with con.begin() as cnx:
|
||||
cnx.execute(drop_datageom)
|
||||
|
||||
|
||||
create_table_pers = """
|
||||
DROP TABLE IF EXISTS md.temp_import_personne;
|
||||
CREATE TABLE md.temp_import_personne AS
|
||||
|
||||
WITH personnel AS (
|
||||
SELECT a.id_personne,a.nom,a.prenom,b.id_structure,b.nom_structure
|
||||
FROM md.personne a
|
||||
JOIN md.structure b USING (id_structure)
|
||||
)
|
||||
SELECT DISTINCT
|
||||
-- personne
|
||||
CASE WHEN observateur2 is null THEN (c.nom||' '||c.prenom)::text
|
||||
WHEN observateur2 is not null AND observateur3 is null THEN (c.nom||' '||c.prenom)||'&'||(d.nom||' '||d.prenom)
|
||||
WHEN observateur3 is not null AND observateur4 is null THEN (c.nom||' '||c.prenom)||'&'||(d.nom||' '||d.prenom)||'&'||(e.nom||' '||e.prenom)
|
||||
WHEN observateur4 is not null AND observateur5 is null THEN (c.nom||' '||c.prenom)||'&'||(d.nom||' '||d.prenom)||'&'||(e.nom||' '||e.prenom)||'&'||(f.nom||' '||f.prenom)
|
||||
|
||||
ELSE c.nom_structure||'&'||d.nom_structure||'&'||(e.nom||' '||e.prenom)||'&'||(f.nom||' '||f.prenom)||'&'||(g.nom||' '||g.prenom)
|
||||
END AS personne,
|
||||
|
||||
-- nom_structure
|
||||
CASE WHEN observateur2 is null THEN c.nom_structure::text
|
||||
WHEN observateur2 is not null AND observateur3 is null AND c.nom_structure <> d.nom_structure
|
||||
THEN c.nom_structure||'&'||d.nom_structure
|
||||
WHEN observateur2 is not null AND observateur3 is null AND c.nom_structure = d.nom_structure
|
||||
THEN c.nom_structure
|
||||
|
||||
WHEN observateur3 is not null AND observateur4 is null AND c.nom_structure <> d.nom_structure AND d.nom_structure <> e.nom_structure
|
||||
AND c.nom_structure <> e.nom_structure
|
||||
THEN c.nom_structure||'&'||d.nom_structure||'&'||e.nom_structure
|
||||
WHEN observateur3 is not null AND observateur4 is null AND c.nom_structure = d.nom_structure AND d.nom_structure <> e.nom_structure
|
||||
THEN c.nom_structure||'&'||e.nom_structure
|
||||
WHEN observateur3 is not null AND observateur4 is null AND c.nom_structure <> d.nom_structure AND d.nom_structure = e.nom_structure
|
||||
THEN c.nom_structure||'&'||d.nom_structure
|
||||
|
||||
WHEN observateur4 is not null AND observateur5 is null AND c.nom_structure <> d.nom_structure
|
||||
THEN c.nom_structure||'&'||d.nom_structure||'&'||e.nom_structure||'&'||f.nom_structure
|
||||
|
||||
ELSE c.nom_structure||'&'||d.nom_structure||'&'||e.nom_structure||'&'||f.nom_structure||'&'||g.nom_structure
|
||||
END AS nom_structure,
|
||||
|
||||
-- observateur
|
||||
CASE WHEN observateur2 is null THEN c.id_personne::text
|
||||
WHEN observateur2 is not null AND observateur3 is null THEN c.id_personne||'&'||d.id_personne
|
||||
WHEN observateur3 is not null AND observateur4 is null THEN c.id_personne||'&'||d.id_personne||'&'||e.id_personne
|
||||
WHEN observateur4 is not null AND observateur5 is null THEN c.id_personne||'&'||d.id_personne||'&'||e.id_personne||'&'||f.id_personne
|
||||
|
||||
ELSE c.id_personne||'&'||d.id_personne||'&'||e.id_personne||'&'||f.id_personne||'&'||g.id_personne
|
||||
END AS observateur,
|
||||
structure
|
||||
FROM import.import_data_cen38 a
|
||||
LEFT JOIN personnel c ON (c.nom||' '||c.prenom) = trim(a.observateur1::text) AND lower(c.nom_structure) = lower(a.structure)
|
||||
LEFT JOIN personnel d ON (d.nom||' '||d.prenom) = trim(a.observateur2::text) AND lower(d.nom_structure) = lower(a.structure)
|
||||
LEFT JOIN personnel e ON (e.nom||' '||e.prenom) = trim(a.observateur3::text) AND lower(e.nom_structure) = lower(a.structure)
|
||||
LEFT JOIN personnel f ON (f.nom||' '||f.prenom) = trim(a.observateur4::text) AND lower(f.nom_structure) = lower(a.structure)
|
||||
LEFT JOIN personnel g ON (g.nom||' '||g.prenom) = trim(a.observateur5::text) AND lower(g.nom_structure) = lower(a.structure)
|
||||
;
|
||||
""".format(id_creator = numerateur.id_personne.item())
|
||||
with con.begin() as cnx:
|
||||
cnx.execute(create_table_pers)
|
||||
|
||||
select_pers = 'SELECT * FROM md.temp_import_personne'
|
||||
df_pers = pd.read_sql_query(
|
||||
sql = select_pers,
|
||||
con = con)
|
||||
|
||||
|
||||
|
||||
|
||||
with_dataTOinsert = """
|
||||
WITH personnel AS (
|
||||
SELECT a.id_personne,a.nom,a.prenom,b.id_structure,b.nom_structure
|
||||
FROM md.personne a
|
||||
JOIN md.structure b USING (id_structure)
|
||||
),
|
||||
t as (
|
||||
SELECT
|
||||
CASE WHEN a.heure IS NULL THEN '00:00:00'
|
||||
ELSE a.heure END heure,
|
||||
a.date_obs, a.date_debut_obs, a.date_fin_obs, a.duree as date_textuelle,
|
||||
CASE WHEN b.regne IS NULL THEN bb.regne
|
||||
ELSE b.regne END regne,
|
||||
CASE WHEN b.phylum IS NULL THEN bb.phylum
|
||||
ELSE b.phylum END phylum,
|
||||
CASE WHEN b.classe IS NULL THEN bb.classe
|
||||
ELSE b.classe END classe,
|
||||
CASE WHEN b.ordre IS NULL THEN bb.ordre
|
||||
ELSE b.ordre END ordre,
|
||||
CASE WHEN b.famille IS NULL THEN bb.famille
|
||||
ELSE b.famille END famille,
|
||||
CASE WHEN b.nom_valide IS NULL THEN bb.nom_valide
|
||||
ELSE b.nom_valide END nom_valide,
|
||||
CASE WHEN b.nom_vern IS NULL THEN bb.nom_vern
|
||||
ELSE b.nom_vern END nom_vern,
|
||||
CASE WHEN b.nom_complet IS NULL THEN bb.nom_complet
|
||||
ELSE b.nom_complet END nom_complet,
|
||||
CASE WHEN b.cd_nom IS NULL THEN bb.cd_nom::text
|
||||
ELSE b.cd_nom END cd_nom,
|
||||
a.abondance_dominance as effectif_textuel, a.effectif_min, a.effectif_max,
|
||||
CASE WHEN strate::text is not null THEN strate::text
|
||||
ELSE age::text END AS type_effectif,
|
||||
CASE WHEN phenologie is not null THEN phenologie
|
||||
ELSE sexe END AS phenologie,
|
||||
|
||||
CASE WHEN latitude::numeric > 6000000
|
||||
THEN ST_X(st_transform(ST_GeomFromText('POINT('||longitude::double precision||' '||latitude::double precision||')',2154),4326)) -- coordonnée X en lambert93
|
||||
WHEN latitude::numeric > 50 AND latitude::numeric < 3000000
|
||||
THEN ST_X(st_transform(ST_GeomFromText('POINT('||longitude::double precision||' '||latitude::double precision||')',27572),4326)) -- coordonnées en Lambert II étendu
|
||||
-- WHEN latitude is null
|
||||
-- THEN ST_X(st_transform(st_pointonsurface(i.geom),4326)) -- pas de coordonnées XY on récupère la geom du centroide du site CenRA si elle existe et si une partie du nom du site est renseignée dans la colonne LOCALISATION
|
||||
ELSE longitude::double precision -- coordonnées en WGS84
|
||||
END as longitude,
|
||||
|
||||
CASE WHEN latitude::numeric > 6000000
|
||||
THEN ST_Y(st_transform(ST_GeomFromText('POINT('||longitude::double precision||' '||latitude::double precision||')',2154),4326)) -- coordonnée X en lambert93
|
||||
WHEN latitude::numeric > 50 AND latitude::numeric < 3000000
|
||||
THEN ST_Y(st_transform(ST_GeomFromText('POINT('||longitude::double precision||' '||latitude::double precision||')',27572),4326)) -- coordonnées en Lambert II étendu
|
||||
-- WHEN latitude is null
|
||||
-- THEN ST_Y(st_transform(st_pointonsurface(i.geom),4326)) -- pas de coordonnées XY on récupère la geom du centroide du site CenRA si elle existe et si une partie du nom du site est renseignée dans la colonne LOCALISATION
|
||||
ELSE latitude::double precision
|
||||
END as latitude,
|
||||
localisation,
|
||||
|
||||
CASE WHEN observateur2 is null THEN c.id_personne::text
|
||||
WHEN observateur2 is not null AND observateur3 is null THEN c.id_personne||'&'||d.id_personne
|
||||
WHEN observateur3 is not null AND observateur4 is null THEN c.id_personne||'&'||d.id_personne||'&'||e.id_personne
|
||||
WHEN observateur4 is not null AND observateur5 is null THEN c.id_personne||'&'||d.id_personne||'&'||e.id_personne||'&'||f.id_personne
|
||||
|
||||
ELSE c.id_personne||'&'||d.id_personne||'&'||e.id_personne||'&'||f.id_personne||'&'||g.id_personne
|
||||
END AS observateur,
|
||||
{id_numerateur}::integer as numerisateur, -- adapter l'identifiant du numérisateur en fonction de la personne qui réalise l'import en masse
|
||||
|
||||
-- cas d'une prestation on ajoute le CenRA à la structure de l'observateur '&1' sinon il faut supprimer
|
||||
CASE WHEN h.id_structure <> 1 THEN (h.id_structure||'&1')::text
|
||||
ELSE h.id_structure::text END as structure,
|
||||
remarque_obs,
|
||||
CASE WHEN lower(a.structure) like '%CEN Isère%' THEN true::boolean
|
||||
ELSE false::boolean END as diffusable,
|
||||
precision::saisie.enum_precision as precision,
|
||||
-- CASE
|
||||
-- WHEN latitude is null AND round(st_maxdistance(st_PointOnSurface(i.geom), st_convexhull(i.geom))::numeric,-2) < 101 THEN '10 à 100m'::saisie.enum_precision
|
||||
-- WHEN latitude is null AND round(st_maxdistance(st_PointOnSurface(i.geom), st_convexhull(i.geom))::numeric,-2) < 501 AND round(st_maxdistance(ST_PointOnSurface(i.geom), st_convexhull(i.geom))::numeric,-2) > 100 THEN '100 à 500m'::saisie.enum_precision
|
||||
-- WHEN latitude is null AND round(st_maxdistance(st_PointOnSurface(i.geom), st_convexhull(i.geom))::numeric,-2) < 1001 AND round(st_maxdistance(ST_PointOnSurface(i.geom), st_convexhull(i.geom))::numeric,-2) > 500 THEN '500 à 1000m'::saisie.enum_precision
|
||||
-- WHEN latitude is null AND round(st_maxdistance(st_PointOnSurface(i.geom), st_convexhull(i.geom))::numeric,-2) > 1001 THEN '> 1000m'::saisie.enum_precision
|
||||
-- ELSE precision::saisie.enum_precision
|
||||
-- END as precision,
|
||||
'validée'::saisie.enum_statut_validation as statut_validation,
|
||||
j.id_etude,
|
||||
k.id_protocole,
|
||||
effectif::integer,
|
||||
CASE WHEN statut_repro::text IS NOT NULL THEN (statut_repro::text)::saisie.enum_reprostatut
|
||||
ELSE NULL END reprostatut,
|
||||
|
||||
CASE WHEN latitude::numeric > 6000000
|
||||
THEN ST_GeomFromText('POINT('||longitude::double precision||' '||latitude::double precision||')',2154) -- coordonnées en lambert93
|
||||
WHEN latitude::numeric > 50 AND latitude::numeric < 3000000
|
||||
THEN ST_transform(ST_GeomFromText('POINT('||longitude::double precision||' '||latitude::double precision||')',27572),2154) -- coordonnées en Lambert II étendu
|
||||
-- WHEN latitude is null
|
||||
-- THEN st_pointonsurface(i.geom) -- pas de coordonnées XY on récupère la geom du centroide du site CenRA si elle existe et si une partie du nom du site est renseignée dans la colonne LOCALISATION
|
||||
ELSE ST_transform(ST_GeomFromText('POINT('||longitude::double precision||' '||latitude::double precision||')',4326),2154) -- coordonnées en WGS84
|
||||
END as geometrie,
|
||||
|
||||
code_releve,
|
||||
CASE WHEN obs_null::text ilike 'oui' THEN true::boolean
|
||||
ELSE false::boolean END as obs_null
|
||||
|
||||
FROM import.import_data_cen38 a
|
||||
LEFT JOIN inpn.taxref b ON b.cd_nom::text = a.cd_nom::text
|
||||
LEFT JOIN inpn.taxref bb ON bb.nom_complet::text = a.nom_complet::text
|
||||
LEFT JOIN personnel c ON (c.nom||' '||c.prenom) = trim(a.observateur1::text) AND lower(c.nom_structure) = lower(a.structure)
|
||||
LEFT JOIN personnel d ON (d.nom||' '||d.prenom) = trim(a.observateur2::text) AND lower(d.nom_structure) = lower(a.structure)
|
||||
LEFT JOIN personnel e ON (e.nom||' '||e.prenom) = trim(a.observateur3::text) AND lower(e.nom_structure) = lower(a.structure)
|
||||
LEFT JOIN personnel f ON (f.nom||' '||f.prenom) = trim(a.observateur4::text) AND lower(f.nom_structure) = lower(a.structure)
|
||||
LEFT JOIN personnel g ON (g.nom||' '||g.prenom) = trim(a.observateur5::text) AND lower(g.nom_structure) = lower(a.structure)
|
||||
JOIN md.structure h ON lower(h.nom_structure) = lower(replace(replace(a.structure,' & ','&'),'CEN38&',''))
|
||||
LEFT JOIN md.etude j on a.id_etude = j.nom_etude
|
||||
LEFT JOIN md.protocole k USING (id_protocole)
|
||||
--LEFT JOIN import.sites_serena_cren_mu_2015 i ON lower(i.nom_site) like '%'||lower(trim(localisation,'s'))||'%' -- on joint la table des sites CenRA pour récupérer éventuellement le centroide d'un site si pas de XY
|
||||
--JOIN md.etude i ON i.nom_etude = a.id_etude -- à utiliser si les champs id_etude et id_protocole contiennent le libelle plutôt que l'identifiant, modifier aussi au-dessus id_etude en i.id_etude, id_protocole en j.id_protocole
|
||||
--JOIN md.protocole j ON j.libelle = a.id_protocole
|
||||
)
|
||||
""".format(id_numerateur=numerateur.id_personne.item())
|
||||
|
||||
select_dataTOinsert = """
|
||||
SELECT DISTINCT
|
||||
heure::TIME WITHOUT TIME ZONE as heure_obs, date_obs, date_debut_obs, date_fin_obs, date_textuelle,
|
||||
regne, nom_vern, nom_complet, cd_nom, effectif_textuel,
|
||||
effectif_min, effectif_max, type_effectif, phenologie,
|
||||
longitude, latitude,
|
||||
localisation,
|
||||
observateur, numerisateur,
|
||||
structure, remarque_obs,
|
||||
c.code_insee,
|
||||
diffusable, "precision",
|
||||
statut_validation,
|
||||
id_etude, id_protocole,
|
||||
effectif, reprostatut,
|
||||
t.geometrie,
|
||||
code_releve id_waypoint,
|
||||
obs_null
|
||||
FROM t, ign_bd_topo.commune c
|
||||
WHERE ST_intersects(t.geometrie ,c.geometrie)
|
||||
--ORDER BY nom_complet, date_obs, longitude, latitude
|
||||
;
|
||||
"""
|
||||
df_data_insert = gpd.read_postgis(
|
||||
sql = text(with_dataTOinsert+select_dataTOinsert),
|
||||
con = con,
|
||||
geom_col='geometrie')
|
||||
# pd.read_sql_query(
|
||||
# sql = text(with_dataTOinsert+select_dataTOinsert),
|
||||
# con = con,)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
if df_data_insert.shape[0] == data_import.shape[0]:
|
||||
insert_data = """
|
||||
INSERT INTO saisie.saisie_observation(
|
||||
heure_obs, date_obs, date_debut_obs, date_fin_obs, date_textuelle, regne, nom_vern, nom_complet, cd_nom, effectif_textuel,
|
||||
effectif_min, effectif_max, type_effectif, phenologie, longitude, latitude, localisation, observateur, numerisateur, structure, remarque_obs, code_insee, diffusable, "precision",
|
||||
statut_validation, id_etude, id_protocole, effectif, reprostatut, geometrie, id_waypoint, obs_null)
|
||||
"""
|
||||
with con.begin() as cnx:
|
||||
cnx.execute(
|
||||
text(with_dataTOinsert+
|
||||
insert_data+
|
||||
select_dataTOinsert)
|
||||
)
|
||||
update_structure_name = """
|
||||
UPDATE saisie.suivi_saisie_observation a
|
||||
SET utilisateur = REPLACE(utilisateur, 'inconnu', '{email_numerator}')
|
||||
WHERE date_operation > '{today}'
|
||||
;
|
||||
""".format(
|
||||
email_numerator=numerateur.email.item(),
|
||||
today = today
|
||||
)
|
||||
drop_temp_table = """
|
||||
DROP TABLE IF EXISTS md.temp_import_personne;
|
||||
"""
|
||||
with con.begin() as cnx:
|
||||
cnx.execute(update_structure_name)
|
||||
cnx.execute(drop_temp_table)
|
||||
else :
|
||||
print(data_import[~data_import.index.isin(df_data_insert.index)])
|
||||
data_import[~data_import.index.isin(data_import.drop_duplicates().index)]
|
||||
|
||||
|
||||
|
||||
check_datageom = """
|
||||
DROP VIEW IF EXISTS import.test_geom_prestation;
|
||||
"""
|
||||
with con.begin() as cnx:
|
||||
cnx.execute(check_datageom)
|
||||
@@ -0,0 +1,457 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-.
|
||||
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.engine import URL
|
||||
from datetime import datetime as dt
|
||||
import pandas as pd
|
||||
import geopandas as gpd
|
||||
from pycen import con_sicen as con
|
||||
|
||||
nom_numerateur = 'GEIER'
|
||||
pnom_numerator = 'Colas'
|
||||
today = dt.now().date().isoformat()
|
||||
|
||||
nom_numerateur = nom_numerateur.upper()
|
||||
pnom_numerator = pnom_numerator[0].upper() + pnom_numerator[1:]
|
||||
query_get_numerator = """
|
||||
SELECT * FROM md.personne
|
||||
WHERE nom = '%s'
|
||||
AND prenom = '%s'
|
||||
AND id_structure = 1
|
||||
""" % (nom_numerateur,pnom_numerator)
|
||||
numerateur = pd.read_sql_query(
|
||||
sql = query_get_numerator,
|
||||
con = con
|
||||
)
|
||||
|
||||
|
||||
select_import = """
|
||||
SELECT * FROM import.import_data_fauneisere_biolovision;
|
||||
"""
|
||||
data_import = pd.read_sql_query(
|
||||
sql = select_import,
|
||||
con = con)
|
||||
|
||||
|
||||
verif_taxref = '''
|
||||
SELECT CASE WHEN c.taxref_cdnom IS NULL THEN bb.cd_nom
|
||||
ELSE c.taxref_cdnom END cdnom_taxref,
|
||||
CASE WHEN c.cd_nom IS NULL THEN cc.cd_nom
|
||||
ELSE c.cd_nom END cd_nom,
|
||||
CASE WHEN c.cd_ref IS NULL THEN cc.cd_ref
|
||||
ELSE c.cd_ref END cd_ref,
|
||||
a.ref, a."id espèce biolovision", a."nom espèce", a."nom latin",
|
||||
a.date, a.jour, a.mois, a.année, a."jour de l'année", a.pentade, a."décade",
|
||||
a."numéro de la semaine", a."heure début", a."heure de début",
|
||||
a."minute de début", a."heure fin", a."heure de fin", a."minute de fin", a.horaire,
|
||||
--a.id_place, a.commune, a.municipality, a.county, a.country, "code insee", a.altitude,
|
||||
a."lat (wgs84)", a."lon (wgs84)", a."type de localisation",
|
||||
a.estimation, a.nombre, a."détails", a."code atlas", a.comportement,
|
||||
a."prénom", a.nom, a."abréviation personne morale",
|
||||
--a.search_export_species_sheet_protection_nat, a.search_export_entity_full_name, a.search_export_entity_tra_full_name,
|
||||
a.search_export_species_sheet_dir_oiseaux, a.search_export_species_sheet_dir_habitat,
|
||||
a.search_export_species_sheet_uicn, a.search_export_species_sheet_lr_nat, a.search_export_species_sheet_znieff,
|
||||
a.search_export_species_sheet_id_fnat,
|
||||
--a."contient des détails mortalité", a.protocol,
|
||||
a.remarque, a."remarque privée",a."date d'insertion", a."date de dernière modification"
|
||||
FROM import.import_data_fauneisere_biolovision a
|
||||
LEFT JOIN (
|
||||
SELECT DISTINCT b.taxref_cdnom, b.id_visio, t.*
|
||||
FROM ref_lpo.taxons_fauneisere_biolovision_taxref b
|
||||
JOIN inpn.taxref t ON t.cd_nom::integer = b.taxref_cdnom) c
|
||||
ON a."id espèce biolovision" = c.id_visio
|
||||
LEFT JOIN (inpn.taxons_isere_absents_taxref bb
|
||||
JOIN inpn.taxref cc ON cc.cd_nom::integer = bb.cd_nom)
|
||||
ON a."id espèce biolovision" = bb.id_taxon
|
||||
;
|
||||
'''
|
||||
df_verif = pd.read_sql_query(
|
||||
sql = verif_taxref,
|
||||
con = con)
|
||||
|
||||
|
||||
if df_verif.shape[0] != data_import.shape[0]:
|
||||
print('shape data_import : %s') % data_import.shape[0]
|
||||
print('shape taxref_verif : %s') % df_verif.shape[0]
|
||||
raise Exception('ERROR : correspondance with TAXREF invalid !')
|
||||
|
||||
# df_verif[df_verif.cd_nom.isna()][["id espèce biolovision","nom espèce","nom latin"]].drop_duplicates()
|
||||
# data_import[~data_import.id_species.isin(df_verif.id_species)] \
|
||||
# [['name_species','latin_species','id_species']].drop_duplicates()
|
||||
|
||||
|
||||
|
||||
update_structure_name = """
|
||||
UPDATE import.import_data_fauneisere_biolovision a
|
||||
SET "abréviation personne morale" = REPLACE("abréviation personne morale", 'CEN38', 'CEN Isère');
|
||||
"""
|
||||
with con.begin() as cnx:
|
||||
cnx.execute(update_structure_name)
|
||||
|
||||
|
||||
|
||||
create_table_pers = """
|
||||
DROP TABLE IF EXISTS md.temp_import_personne;
|
||||
CREATE TABLE md.temp_import_personne AS
|
||||
|
||||
WITH nom_complet_identifies AS
|
||||
(SELECT DISTINCT
|
||||
ltrim(unnest(string_to_array(a."prénom",'&'))) as prenom,
|
||||
ltrim(unnest(CASE WHEN "abréviation personne morale" IN ('CEN Isère','-') THEN string_to_array(''|| upper(split_part(a.nom,' /',1)),'&')
|
||||
ELSE string_to_array(upper(split_part(a.nom,' (',1)),'&') END)) as nom,
|
||||
CASE WHEN "abréviation personne morale" ILIKE '-' THEN 'CEN Isère'
|
||||
ELSE "abréviation personne morale" END as structure,
|
||||
a.nom || ' ' || a."prénom" as full_name
|
||||
FROM import.import_data_fauneisere_biolovision a
|
||||
ORDER BY full_name),
|
||||
|
||||
personne_sicen AS
|
||||
(SELECT a.id_personne, prenom, nom, nom || ' ' || prenom as nom_complet, role, specialite,
|
||||
a.id_structure, b.nom_structure
|
||||
FROM md.personne a
|
||||
JOIN md.structure b ON a.id_structure = b.id_structure
|
||||
--WHERE personne.id_personne BETWEEN 100000 AND 999998
|
||||
)
|
||||
|
||||
SELECT DISTINCT
|
||||
CASE WHEN b.id_personne IS NULL THEN nextval('md.personne_id_personne_hors_cen38_seq'::regclass)
|
||||
ELSE b.id_personne END as id_personne,
|
||||
b.id_personne id_personne_exist,
|
||||
'Observateur Faune-Isère' as remarque,
|
||||
a.prenom as prenom,
|
||||
a.nom as nom,
|
||||
a.full_name as nom_complet,
|
||||
b.nom_structure,
|
||||
a.structure,
|
||||
CASE WHEN a.structure ILIKE 'CEN Isère' THEN 1
|
||||
ELSE 1 END AS id_structure
|
||||
FROM nom_complet_identifies a
|
||||
LEFT JOIN personne_sicen b ON lower(a.nom || ' ' || a.prenom) = lower(b.nom_complet) AND b.nom_structure = a.structure
|
||||
;
|
||||
"""
|
||||
with con.begin() as cnx:
|
||||
cnx.execute(create_table_pers)
|
||||
|
||||
select_pers = 'SELECT * FROM md.temp_import_personne'
|
||||
df_pers = pd.read_sql_query(
|
||||
sql = select_pers,
|
||||
con = con)
|
||||
|
||||
|
||||
select_persTOinsert = """
|
||||
SELECT DISTINCT
|
||||
a.id_personne,
|
||||
a.remarque,
|
||||
a.prenom as prenom,
|
||||
a.nom as nom, --full_name
|
||||
NULL as email,
|
||||
'observ'::md.enum_role,
|
||||
NULL::md.enum_specialite as specialite,
|
||||
0 as mot_de_passe,
|
||||
{id_creator} as createur,
|
||||
'Mme / M.'::md.enum_titre as titre,
|
||||
now()::date as date_maj,
|
||||
a.id_structure
|
||||
FROM md.temp_import_personne a
|
||||
WHERE NOT EXISTS (SELECT DISTINCT id_personne FROM md.personne)
|
||||
;
|
||||
""".format(id_creator = numerateur.id_personne.item())
|
||||
df_pers_insert = pd.read_sql_query(
|
||||
sql = select_persTOinsert,
|
||||
con = con)
|
||||
|
||||
if not df_pers_insert.empty:
|
||||
raise Exception("DEV en cours ...")
|
||||
insert_pers = """
|
||||
INSERT INTO md.personne(
|
||||
id_personne,
|
||||
remarque,
|
||||
prenom,
|
||||
nom,
|
||||
email,
|
||||
role,
|
||||
specialite,
|
||||
mot_de_passe,
|
||||
createur,
|
||||
titre,
|
||||
date_maj,
|
||||
id_structure)
|
||||
|
||||
""" + select_persTOinsert
|
||||
with con.begin() as cnx:
|
||||
cnx.execute(insert_pers)
|
||||
else:
|
||||
pass
|
||||
|
||||
|
||||
select_pers = """
|
||||
WITH observateurs_multiples AS
|
||||
(SELECT DISTINCT
|
||||
string_agg(a.id_personne::text,'&') as id_personne,
|
||||
--a.remarque,
|
||||
--a.prenom as prenom,
|
||||
--a.nom as nom,
|
||||
a.nom_complet,
|
||||
a.id_structure,
|
||||
s.nom_structure
|
||||
FROM md.temp_import_personne a
|
||||
LEFT JOIN md.structure s ON a.id_structure = s.id_structure
|
||||
GROUP BY a.nom_complet, a.id_structure,s.nom_structure)
|
||||
"""
|
||||
|
||||
select_dataTOinsert = """
|
||||
SELECT DISTINCT
|
||||
--a."id espèce biolovision",
|
||||
--a.name_species,
|
||||
--a.latin_species,
|
||||
--a.protegee,
|
||||
--a.embr,
|
||||
--a.classe1,
|
||||
--a.classe2,
|
||||
a.date::date,
|
||||
CASE WHEN a.horaire <> '00:00:00' THEN a.horaire::time
|
||||
ELSE NULL END as heure_obs,
|
||||
--a.date_year,
|
||||
--a.time_start,
|
||||
--a.time_stop,
|
||||
CASE WHEN t.regne IS NULL THEN tt.regne
|
||||
ELSE t.regne END regne,
|
||||
CASE WHEN t.phylum IS NULL THEN tt.phylum
|
||||
ELSE t.phylum END phylum,
|
||||
CASE WHEN t.classe IS NULL THEN tt.classe
|
||||
ELSE t.classe END classe,
|
||||
CASE WHEN t.ordre IS NULL THEN tt.ordre
|
||||
ELSE t.ordre END ordre,
|
||||
CASE WHEN t.famille IS NULL THEN tt.famille
|
||||
ELSE t.famille END famille,
|
||||
CASE WHEN t.nom_valide IS NULL THEN tt.nom_valide
|
||||
ELSE t.nom_valide END nom_valide,
|
||||
CASE WHEN t.nom_vern IS NULL THEN tt.nom_vern
|
||||
ELSE t.nom_vern END nom_vern,
|
||||
CASE WHEN t.nom_complet IS NULL THEN tt.nom_complet
|
||||
ELSE t.nom_complet END nom_complet,
|
||||
CASE WHEN t.cd_nom IS NULL THEN tt.cd_nom
|
||||
ELSE t.cd_nom END cd_nom,
|
||||
--t.cd_nom cd_nom_2,
|
||||
--b.taxref_cdnom,
|
||||
CASE WHEN a.estimation IS NULL AND (a.nombre IS NOT NULL OR a.nombre > 0) THEN a.nombre
|
||||
WHEN a.estimation LIKE 'x' THEN '1'
|
||||
ELSE NULL END AS effectif,
|
||||
CASE WHEN a.estimation LIKE '>' AND (a.nombre IS NOT NULL OR a.nombre > 0) THEN a.nombre
|
||||
WHEN a.estimation LIKE '~' AND (a.nombre IS NOT NULL OR a.nombre > 0) THEN a.nombre
|
||||
ELSE NULL END AS effectif_min,
|
||||
CASE WHEN a.estimation LIKE '~' AND (a.nombre IS NOT NULL OR a.nombre > 0) THEN (a.nombre + 10)
|
||||
ELSE NULL END AS effectif_max,
|
||||
a."lon (wgs84)" as longitude,
|
||||
a."lat (wgs84)" as latitude,
|
||||
a.commune as localisation,
|
||||
o.id_personne as observateur,
|
||||
/*a."prénom",
|
||||
a.nom,
|
||||
a.entity_short_name,
|
||||
a.entity_full_name,
|
||||
a.tra_surname,
|
||||
a.tra_name,
|
||||
a.tra_full_name,*/
|
||||
{id_numerateur} as numerisateur,
|
||||
o.id_structure,
|
||||
concat_ws(' | ',
|
||||
CASE WHEN a."détails" IS NOT NULL THEN 'Détails : '|| a."détails" ELSE NULL END,
|
||||
CASE WHEN a."contient des détails mortalité" ILIKE 'oui' THEN 'Mortalité' ELSE NULL END,
|
||||
CASE WHEN a.comportement IS NOT NULL THEN 'Comportement : '|| a.comportement ELSE NULL END,
|
||||
CASE WHEN a."code atlas" IS NOT NULL THEN 'Code atlas : '|| c.categorie || CONCAT(' (',c.libelle,')') ELSE NULL END,
|
||||
CASE WHEN a.remarque IS NOT NULL THEN 'Commentaires : '|| a.remarque ELSE NULL END
|
||||
) as remarque_obs,
|
||||
--a.id_place,
|
||||
--a.municipality,
|
||||
--a.county,
|
||||
--a.country,
|
||||
"code insee" as code_insee,
|
||||
TRUE as diffusable,
|
||||
--CASE WHEN a.hidden ILIKE 'oui' THEN TRUE ELSE FALSE END as diffusable,
|
||||
CASE WHEN lower(a."type de localisation") ILIKE 'lieu-dit' THEN 'lieu-dit'
|
||||
WHEN lower(a."type de localisation") ILIKE 'localisation précise' THEN '0 à 10m'
|
||||
ELSE NULL END::saisie.enum_precision AS "precision",
|
||||
--a.grid_name,
|
||||
--a.estimation,
|
||||
'validée'::saisie.enum_statut_validation as statut_validation,
|
||||
99 as id_etude, -- Echange de données (partenaires)
|
||||
3 as id_protocole, -- Observation aléatoire
|
||||
a.altitude as elevation,
|
||||
--a.hidden,
|
||||
--a.admin_hidden,
|
||||
--a.committee_chn,
|
||||
--a.committee_chr,
|
||||
--a.second_hand,
|
||||
--a.private_comment,
|
||||
--a.insert_date,
|
||||
--a.update_date,
|
||||
--a.project_code,
|
||||
ST_Transform(st_setsrid(st_makepoint("lon (wgs84)", "lat (wgs84)"),4326),2154)::geometry(POINT,2154) AS geometrie,
|
||||
CASE WHEN (a.nombre IS NULL OR a.nombre = 0) THEN TRUE ELSE FALSE END AS obs_null,
|
||||
8 as id_lot, -- Code du lot de données : Import Faune-Isère (exporté par LPO)
|
||||
a.ref as id_origine
|
||||
|
||||
FROM import.import_data_fauneisere_biolovision a
|
||||
LEFT JOIN (
|
||||
SELECT DISTINCT b.taxref_cdnom, b.id_visio, t.*
|
||||
FROM ref_lpo.taxons_fauneisere_biolovision_taxref b
|
||||
JOIN inpn.taxref t ON t.cd_nom::integer = b.taxref_cdnom) t
|
||||
ON a."id espèce biolovision" = t.id_visio
|
||||
LEFT JOIN (inpn.taxons_isere_absents_taxref bb
|
||||
JOIN inpn.taxref tt ON tt.cd_nom::integer = bb.cd_nom)
|
||||
ON a."id espèce biolovision" = bb.id_taxon
|
||||
LEFT JOIN ref_lpo.tr_code_atlas_lpo c ON a."code atlas" = c.code_atlas
|
||||
JOIN observateurs_multiples o ON lower(a.nom || ' ' || a."prénom") = lower(o.nom_complet)
|
||||
--LEFT JOIN saisie.saisie_observation s
|
||||
--ON ((a.ref::text = s.id_origine) AND (id_lot IN (7.8)))
|
||||
--AND s.id_origine IS NULL
|
||||
;
|
||||
""".format(id_numerateur=numerateur.id_personne.item())
|
||||
df_data_insert = gpd.read_postgis(
|
||||
sql = text(select_pers+select_dataTOinsert),
|
||||
con = con,
|
||||
geom_col='geometrie')
|
||||
|
||||
# df_data_insert[df_data_insert.id_origine.duplicated(keep=False)]
|
||||
# df_data_insert[df_data_insert.id_origine.duplicated(keep=False)].cd_nom.unique()
|
||||
# df_data_insert[df_data_insert.cd_nom.isna()]
|
||||
|
||||
if df_data_insert.shape[0] == data_import.shape[0]:
|
||||
insert_data = """
|
||||
INSERT INTO saisie.saisie_observation(
|
||||
--id_obs,
|
||||
date_obs,
|
||||
--date_debut_obs,
|
||||
--date_fin_obs,
|
||||
heure_obs,
|
||||
--date_textuelle,
|
||||
regne,
|
||||
phylum,
|
||||
classe,
|
||||
ordre,
|
||||
famille,
|
||||
nom_valide,
|
||||
nom_vern,
|
||||
nom_complet,
|
||||
cd_nom,
|
||||
--effectif_textuel,
|
||||
effectif,
|
||||
effectif_min,
|
||||
effectif_max,
|
||||
--type_effectif,
|
||||
--phenologie,
|
||||
--id_waypoint,
|
||||
longitude,
|
||||
latitude,
|
||||
localisation,
|
||||
observateur,
|
||||
numerisateur,
|
||||
--validateur,
|
||||
structure,
|
||||
remarque_obs,
|
||||
code_insee,
|
||||
--id_lieu_dit,
|
||||
diffusable,
|
||||
"precision",
|
||||
statut_validation,
|
||||
id_etude,
|
||||
id_protocole,
|
||||
--url_photo,
|
||||
--commentaire_photo,
|
||||
--decision_validation,
|
||||
--determination,
|
||||
elevation,
|
||||
geometrie,
|
||||
--qualification,
|
||||
--reprostatut,
|
||||
obs_null,
|
||||
--uuid,
|
||||
id_lot,
|
||||
id_origine)
|
||||
"""
|
||||
with con.begin() as cnx:
|
||||
cnx.execute(
|
||||
text(select_pers+
|
||||
insert_data+
|
||||
select_dataTOinsert)
|
||||
)
|
||||
|
||||
|
||||
update_structure_name = """
|
||||
UPDATE saisie.suivi_saisie_observation a
|
||||
SET utilisateur = REPLACE(utilisateur, 'inconnu', '{email_numerator}')
|
||||
WHERE date_operation > '{today}'
|
||||
;
|
||||
""".format(
|
||||
email_numerator=numerateur.email.item(),
|
||||
today = today
|
||||
)
|
||||
drop_temp_table = """
|
||||
DROP TABLE IF EXISTS md.temp_import_personne;
|
||||
"""
|
||||
with con.begin() as cnx:
|
||||
cnx.execute(update_structure_name)
|
||||
cnx.execute(drop_temp_table)
|
||||
|
||||
|
||||
|
||||
{
|
||||
'ref': 'ref',
|
||||
"id espèce biolovision": 'id_species',
|
||||
"lat (wgs84)": 'coord_lat',
|
||||
"lon (wgs84)": 'coord_lon',
|
||||
"nom espèce": 'name_species',
|
||||
"nom latin": 'latin_species',
|
||||
# 'abréviation de la personne morale du transmetteur',
|
||||
"prénom":'surname',
|
||||
'nom':'name',
|
||||
'abréviation personne morale':'search_export_entity_short_name',
|
||||
# 'personne morale',
|
||||
'nombre': 'total_count',
|
||||
'estimation': 'estimation_code',
|
||||
'jour':'date_day',
|
||||
'mois':'date_month',
|
||||
'année':'date_year',
|
||||
"jour de l'année":'date_jday',
|
||||
'pentade':'date_pentade',
|
||||
'décade':'date_decade',
|
||||
"numéro de la semaine":'date_week',
|
||||
"heure fin":'time_stop',
|
||||
"heure de fin":'time_stop_hour',
|
||||
"minute de fin":'time_stop_min',
|
||||
"heure début":'time_start',
|
||||
"heure de début":'time_start_hour',
|
||||
"minute de début":'time_start_min',
|
||||
# 'id universel transmetteur',
|
||||
# 'id transmetteur',
|
||||
# 'e-mail',
|
||||
# 'personne morale du transmetteur',
|
||||
# 'id universel observateur',
|
||||
# 'nom transmetteur',
|
||||
# 'prénom transmetteur',
|
||||
# 'transmetteur anonyme',
|
||||
# 'anonyme',
|
||||
# 'liste complète ?',
|
||||
# 'protégée',
|
||||
# 'vérification',
|
||||
# 'donnée de seconde main',
|
||||
# 'commentaire de la liste',
|
||||
# 'ordre systématique',
|
||||
"détails": 'detail',
|
||||
'horaire':'timing',
|
||||
"type de localisation": 'precision',
|
||||
'comportement': 'behaviour',
|
||||
'remarque': 'comment',
|
||||
"remarque privée": 'private_comment',
|
||||
# 'protection nationale',
|
||||
"date d'insertion": 'insert_date',
|
||||
"date de dernière modification": 'update_date',
|
||||
"code atlas":'atlas_code',
|
||||
"code insee": 'insee',
|
||||
"altitude": 'altitude',
|
||||
"commune": 'place',
|
||||
"contient des détails mortalité":'has_death_info',
|
||||
# 'id liste'
|
||||
}
|
||||
@@ -0,0 +1,391 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-.
|
||||
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.engine import URL
|
||||
from datetime import datetime as dt
|
||||
import pandas as pd
|
||||
import geopandas as gpd
|
||||
from pycen import con_sicen as con
|
||||
|
||||
nom_numerateur = 'GEIER'
|
||||
pnom_numerator = 'Colas'
|
||||
today = dt.now().date().isoformat()
|
||||
|
||||
nom_numerateur = nom_numerateur.upper()
|
||||
pnom_numerator = pnom_numerator[0].upper() + pnom_numerator[1:]
|
||||
query_get_numerator = """
|
||||
SELECT * FROM md.personne
|
||||
WHERE nom = '%s'
|
||||
AND prenom = '%s'
|
||||
AND id_structure = 1
|
||||
""" % (nom_numerateur,pnom_numerator)
|
||||
numerateur = pd.read_sql_query(
|
||||
sql = query_get_numerator,
|
||||
con = con
|
||||
)
|
||||
|
||||
create_view='''
|
||||
DROP VIEW import.test_import_faune_isere;
|
||||
# CREATE OR REPLACE VIEW import.test_import_faune_isere AS
|
||||
# SELECT DISTINCT
|
||||
# *,
|
||||
# st_setsrid(st_makepoint(coord_lon_l93, coord_lat_l93),2154)::geometry(POINT,2154) AS geom
|
||||
# FROM import.import_data_fauneisere;
|
||||
'''
|
||||
with con.begin() as cnx:
|
||||
cnx.execute(create_view)
|
||||
|
||||
|
||||
select_import = """
|
||||
SELECT * FROM import.import_data_fauneisere;
|
||||
"""
|
||||
data_import = pd.read_sql_query(
|
||||
sql = select_import,
|
||||
con = con)
|
||||
|
||||
|
||||
verif_taxref = '''
|
||||
SELECT CASE WHEN b.cdnom_taxref IS NULL THEN bb.cd_nom
|
||||
ELSE b.cdnom_taxref END cdnom_taxref,
|
||||
b.cdnom_taxref,
|
||||
CASE WHEN c.cd_nom IS NULL THEN cc.cd_nom
|
||||
ELSE c.cd_nom END cd_nom,
|
||||
--c.cd_nom,
|
||||
CASE WHEN c.cd_ref IS NULL THEN cc.cd_ref
|
||||
ELSE c.cd_ref END cd_ref,
|
||||
--c.cd_ref,
|
||||
a.id_sighting, a.id_species, a.name_species, a.id_observer, a.latin_species,
|
||||
a.taxonomy_name, a.family_name, a.sys_order, a.date, a.date_day, a.date_month, a.date_year,
|
||||
a.date_jday, a.date_pentade, a.date_decade, a.date_week, a.time_start, a.time_start_hour, a.time_start_min, a.time_stop, a.time_stop_hour, a.time_stop_min,
|
||||
a.full_form, a.timing, a.id_place, a.place, a.municipality, a.county, a.country, a.insee, a.coord_lon_liie, a.coord_lat_liie, a.coord_lon_l93, a.coord_lat_l93,
|
||||
a.coord_lat, a.coord_lon, a.coord_lat_dms, a.coord_lon_dms, a.coord_f, a.coord_e, a.coord_n, a."precision", a.altitude, a.grid_name,
|
||||
a.estimation_code, a.total_count,
|
||||
a.detail, a.atlas_code,
|
||||
a.hidden, a.admin_hidden, a.behaviour,
|
||||
--a.committee_chn, a.committee_chr,
|
||||
a.search_export_species_sheet_dir_oiseaux, a.search_export_species_sheet_protection_nat, a.search_export_species_sheet_dir_habitat,
|
||||
a.search_export_species_sheet_uicn, a.search_export_species_sheet_lr_nat, a.search_export_species_sheet_znieff, a.search_export_species_sheet_id_fnat,
|
||||
a.surname, a.name, a.search_export_entity_short_name, a.search_export_entity_full_name, a.tra_surname, a.tra_name, a.id_form,
|
||||
a.search_export_entity_tra_full_name, a.second_hand, a.comment, a.private_comment, a.daily_text_comment_rem, a.insert_date, a.update_date, a.protocol, a.has_death_info
|
||||
FROM import.import_data_fauneisere a
|
||||
LEFT JOIN (ref_lpo.taxons_faune_isere_taxref b
|
||||
JOIN inpn.taxref c ON c.cd_nom::integer = b.cdnom_taxref)
|
||||
ON a.latin_species = b.latin_species
|
||||
LEFT JOIN (inpn.taxons_isere_absents_taxref bb
|
||||
JOIN inpn.taxref cc ON cc.cd_nom::integer = bb.cd_nom)
|
||||
ON a.id_species = bb.id_taxon
|
||||
;
|
||||
'''
|
||||
df_verif = pd.read_sql_query(
|
||||
sql = verif_taxref,
|
||||
con = con)
|
||||
|
||||
data_import[~data_import.id_species.isin(df_verif.id_species)] \
|
||||
[['name_species','latin_species','id_species']].drop_duplicates()
|
||||
|
||||
|
||||
|
||||
update_structure_name = """
|
||||
UPDATE import.import_data_fauneisere a
|
||||
SET search_export_entity_short_name = REPLACE(search_export_entity_short_name, 'CEN38', 'CEN Isère');
|
||||
"""
|
||||
with con.begin() as cnx:
|
||||
cnx.execute(update_structure_name)
|
||||
|
||||
|
||||
|
||||
create_table_pers = """
|
||||
DROP TABLE IF EXISTS md.temp_import_personne;
|
||||
CREATE TABLE md.temp_import_personne AS
|
||||
|
||||
WITH nom_complet_identifies AS
|
||||
(SELECT DISTINCT
|
||||
ltrim(unnest(string_to_array(a.surname,'&'))) as prenom,
|
||||
ltrim(unnest(CASE WHEN search_export_entity_short_name IN ('CEN Isère','-') THEN string_to_array(''|| upper(split_part(a.name,' /',1)),'&')
|
||||
ELSE string_to_array(upper(split_part(a.name,' (',1)),'&') END)) as nom,
|
||||
CASE WHEN search_export_entity_short_name ILIKE '-' THEN 'CEN Isère'
|
||||
ELSE search_export_entity_short_name END as structure,
|
||||
a.name || ' ' || a.surname as full_name
|
||||
FROM import.import_data_fauneisere a
|
||||
ORDER BY full_name),
|
||||
|
||||
personne_sicen AS
|
||||
(SELECT a.id_personne, prenom, nom, nom || ' ' || prenom as nom_complet, role, specialite,
|
||||
a.id_structure, b.nom_structure
|
||||
FROM md.personne a
|
||||
JOIN md.structure b ON a.id_structure = b.id_structure
|
||||
--WHERE personne.id_personne BETWEEN 100000 AND 999998
|
||||
)
|
||||
|
||||
SELECT DISTINCT
|
||||
CASE WHEN b.id_personne IS NULL THEN nextval('md.personne_id_personne_hors_cen38_seq'::regclass)
|
||||
ELSE b.id_personne END as id_personne,
|
||||
b.id_personne id_personne_exist,
|
||||
'Observateur Faune-Isère' as remarque,
|
||||
a.prenom as prenom,
|
||||
a.nom as nom,
|
||||
a.full_name as nom_complet,
|
||||
b.nom_structure,
|
||||
a.structure,
|
||||
CASE WHEN a.structure ILIKE 'CEN Isère' THEN 1
|
||||
ELSE 1 END AS id_structure
|
||||
FROM nom_complet_identifies a
|
||||
LEFT JOIN personne_sicen b ON lower(a.nom || ' ' || a.prenom) = lower(b.nom_complet) AND b.nom_structure = a.structure
|
||||
;
|
||||
"""
|
||||
with con.begin() as cnx:
|
||||
cnx.execute(create_table_pers)
|
||||
|
||||
|
||||
select_persTOinsert = """
|
||||
SELECT DISTINCT
|
||||
a.id_personne,
|
||||
a.remarque,
|
||||
a.prenom as prenom,
|
||||
a.nom as nom, --full_name
|
||||
NULL as email,
|
||||
'observ'::md.enum_role,
|
||||
NULL::md.enum_specialite as specialite,
|
||||
0 as mot_de_passe,
|
||||
{id_creator} as createur,
|
||||
'Mme / M.'::md.enum_titre as titre,
|
||||
now()::date as date_maj,
|
||||
a.id_structure
|
||||
FROM md.temp_import_personne a
|
||||
WHERE NOT EXISTS (SELECT DISTINCT id_personne FROM md.personne)
|
||||
;
|
||||
""".format(id_creator = numerateur.id_personne.item())
|
||||
df_pers_insert = pd.read_sql_query(
|
||||
sql = select_persTOinsert,
|
||||
con = con)
|
||||
|
||||
if not df_pers_insert.empty:
|
||||
raise Exception("DEV en cours ...")
|
||||
insert_pers = """
|
||||
INSERT INTO md.personne(
|
||||
id_personne,
|
||||
remarque,
|
||||
prenom,
|
||||
nom,
|
||||
email,
|
||||
role,
|
||||
specialite,
|
||||
mot_de_passe,
|
||||
createur,
|
||||
titre,
|
||||
date_maj,
|
||||
id_structure)
|
||||
|
||||
""" + select_persTOinsert
|
||||
with con.begin() as cnx:
|
||||
cnx.execute(insert_pers)
|
||||
else:
|
||||
pass
|
||||
|
||||
|
||||
select_pers = """
|
||||
WITH observateurs_multiples AS
|
||||
(SELECT DISTINCT
|
||||
string_agg(a.id_personne::text,'&') as id_personne,
|
||||
--a.remarque,
|
||||
--a.prenom as prenom,
|
||||
--a.nom as nom,
|
||||
a.nom_complet,
|
||||
a.id_structure,
|
||||
s.nom_structure
|
||||
FROM md.temp_import_personne a
|
||||
LEFT JOIN md.structure s ON a.id_structure = s.id_structure
|
||||
GROUP BY a.nom_complet, a.id_structure,s.nom_structure)
|
||||
"""
|
||||
|
||||
select_dataTOinsert = """
|
||||
SELECT DISTINCT
|
||||
--a.id_species,
|
||||
--a.name_species,
|
||||
--a.latin_species,
|
||||
--a.protegee,
|
||||
--a.embr,
|
||||
--a.classe1,
|
||||
--a.classe2,
|
||||
a.date::date,
|
||||
CASE WHEN a.timing <> '00:00:00' THEN a.timing::time
|
||||
ELSE NULL END as heure_obs,
|
||||
--a.date_year,
|
||||
a.time_start,
|
||||
a.time_stop,
|
||||
CASE WHEN t.regne IS NULL THEN tt.regne
|
||||
ELSE t.regne END regne,
|
||||
CASE WHEN t.phylum IS NULL THEN tt.phylum
|
||||
ELSE t.phylum END phylum,
|
||||
CASE WHEN t.classe IS NULL THEN tt.classe
|
||||
ELSE t.classe END classe,
|
||||
CASE WHEN t.ordre IS NULL THEN tt.ordre
|
||||
ELSE t.ordre END ordre,
|
||||
CASE WHEN t.famille IS NULL THEN tt.famille
|
||||
ELSE t.famille END famille,
|
||||
CASE WHEN t.nom_valide IS NULL THEN tt.nom_valide
|
||||
ELSE t.nom_valide END nom_valide,
|
||||
CASE WHEN t.nom_vern IS NULL THEN tt.nom_vern
|
||||
ELSE t.nom_vern END nom_vern,
|
||||
CASE WHEN t.nom_complet IS NULL THEN tt.nom_complet
|
||||
ELSE t.nom_complet END nom_complet,
|
||||
CASE WHEN t.cd_nom IS NULL THEN tt.cd_nom
|
||||
ELSE t.cd_nom END cd_nom,
|
||||
--t.cd_nom cd_nom_2,
|
||||
--b.cdnom_taxref,
|
||||
CASE WHEN a.estimation_code IS NULL AND (a.total_count IS NOT NULL OR a.total_count > 0) THEN a.total_count
|
||||
WHEN a.estimation_code LIKE 'x' THEN '1'
|
||||
ELSE NULL END AS effectif,
|
||||
CASE WHEN a.estimation_code LIKE '>' AND (a.total_count IS NOT NULL OR a.total_count > 0) THEN a.total_count
|
||||
WHEN a.estimation_code LIKE '~' AND (a.total_count IS NOT NULL OR a.total_count > 0) THEN a.total_count
|
||||
ELSE NULL END AS effectif_min,
|
||||
CASE WHEN a.estimation_code LIKE '~' AND (a.total_count IS NOT NULL OR a.total_count > 0) THEN (a.total_count + 10)
|
||||
ELSE NULL END AS effectif_max,
|
||||
a.coord_lon_l93 as longitude,
|
||||
a.coord_lat_l93 as latitude,
|
||||
a.place as localisation,
|
||||
o.id_personne as observateur,
|
||||
/*a.surname,
|
||||
a.name,
|
||||
a.entity_short_name,
|
||||
a.entity_full_name,
|
||||
a.tra_surname,
|
||||
a.tra_name,
|
||||
a.tra_full_name,*/
|
||||
{id_numerateur} as numerisateur,
|
||||
o.id_structure,
|
||||
concat_ws(' | ',
|
||||
CASE WHEN a.detail IS NOT NULL THEN 'Détails : '|| a.detail ELSE NULL END,
|
||||
--CASE WHEN a.has_death_info ILIKE 'oui' THEN 'Mortalité' ELSE NULL END,
|
||||
CASE WHEN a.behaviour IS NOT NULL THEN 'Comportement : '|| a.behaviour ELSE NULL END,
|
||||
CASE WHEN a.atlas_code IS NOT NULL THEN 'Code atlas : '|| a.atlas_code ELSE NULL END,
|
||||
CASE WHEN a.comment IS NOT NULL THEN 'Commentaires : '|| a.comment ELSE NULL END
|
||||
) as remarque_obs,
|
||||
--a.id_place,
|
||||
--a.municipality,
|
||||
--a.county,
|
||||
--a.country,
|
||||
a.insee as code_insee,
|
||||
TRUE as diffusable,
|
||||
--CASE WHEN a.hidden ILIKE 'oui' THEN TRUE ELSE FALSE END as diffusable,
|
||||
CASE WHEN lower(a."precision") ILIKE 'lieu-dit' THEN 'lieu-dit'
|
||||
WHEN lower(a."precision") ILIKE 'localisation précise' THEN '0 à 10m'
|
||||
ELSE NULL END::saisie.enum_precision AS "precision",
|
||||
--a.grid_name,
|
||||
--a.estimation_code,
|
||||
'validée'::saisie.enum_statut_validation as statut_validation,
|
||||
99 as id_etude, -- Echange de données (partenaires)
|
||||
3 as id_protocole, -- Observation aléatoire
|
||||
a.altitude as elevation,
|
||||
--a.hidden,
|
||||
--a.admin_hidden,
|
||||
--a.committee_chn,
|
||||
--a.committee_chr,
|
||||
--a.second_hand,
|
||||
--a.private_comment,
|
||||
--a.insert_date,
|
||||
--a.update_date,
|
||||
--a.project_code,
|
||||
st_setsrid(st_makepoint(coord_lon_l93, coord_lat_l93),2154)::geometry(POINT,2154) AS geometrie,
|
||||
CASE WHEN (a.total_count IS NULL OR a.total_count = 0) THEN TRUE ELSE FALSE END AS obs_null,
|
||||
8 as id_lot, -- Code du lot de données : Import Faune-Isère (exporté par LPO)
|
||||
a.id_sighting as id_origine
|
||||
|
||||
FROM import.import_data_fauneisere a
|
||||
LEFT JOIN (ref_lpo.taxons_faune_isere_taxref b
|
||||
JOIN inpn.taxref t ON t.cd_nom::integer = b.cdnom_taxref)
|
||||
ON a.latin_species = b.latin_species
|
||||
LEFT JOIN (inpn.taxons_isere_absents_taxref bb
|
||||
JOIN inpn.taxref tt ON tt.cd_nom::integer = bb.cd_nom)
|
||||
ON a.id_species = bb.id_taxon
|
||||
JOIN observateurs_multiples o ON lower(a.name || ' ' || a.surname) = lower(o.nom_complet)
|
||||
--LEFT JOIN saisie.saisie_observation s
|
||||
--ON ((a.id_sighting::text = s.id_origine) AND (id_lot IN (7.8)))
|
||||
--AND s.id_origine IS NULL
|
||||
;
|
||||
""".format(id_numerateur=numerateur.id_personne.item())
|
||||
df_data_insert = gpd.read_postgis(
|
||||
sql = text(select_pers+select_dataTOinsert),
|
||||
con = con,
|
||||
geom_col='geometrie')
|
||||
|
||||
|
||||
if df_data_insert.shape[0] == data_import.shape[0]:
|
||||
insert_data = """
|
||||
INSERT INTO saisie.saisie_observation(
|
||||
--id_obs,
|
||||
date_obs,
|
||||
--date_debut_obs,
|
||||
--date_fin_obs,
|
||||
heure_obs,
|
||||
--date_textuelle,
|
||||
regne,
|
||||
phylum,
|
||||
classe,
|
||||
ordre,
|
||||
famille,
|
||||
nom_valide,
|
||||
nom_vern,
|
||||
nom_complet,
|
||||
cd_nom,
|
||||
--effectif_textuel,
|
||||
effectif,
|
||||
effectif_min,
|
||||
effectif_max,
|
||||
--type_effectif,
|
||||
--phenologie,
|
||||
--id_waypoint,
|
||||
longitude,
|
||||
latitude,
|
||||
localisation,
|
||||
observateur,
|
||||
numerisateur,
|
||||
--validateur,
|
||||
structure,
|
||||
remarque_obs,
|
||||
code_insee,
|
||||
--id_lieu_dit,
|
||||
diffusable,
|
||||
"precision",
|
||||
statut_validation,
|
||||
id_etude,
|
||||
id_protocole,
|
||||
--url_photo,
|
||||
--commentaire_photo,
|
||||
--decision_validation,
|
||||
--determination,
|
||||
elevation,
|
||||
geometrie,
|
||||
--qualification,
|
||||
--reprostatut,
|
||||
obs_null,
|
||||
--uuid,
|
||||
id_lot,
|
||||
id_origine)
|
||||
|
||||
"""
|
||||
with con.begin() as cnx:
|
||||
cnx.execute(
|
||||
text(select_pers+
|
||||
insert_data+
|
||||
select_dataTOinsert)
|
||||
)
|
||||
|
||||
|
||||
update_structure_name = """
|
||||
UPDATE saisie.suivi_saisie_observation a
|
||||
SET utilisateur = REPLACE(utilisateur, 'inconnu', '{email_numerator}')
|
||||
WHERE date_operation > '{today}'
|
||||
;
|
||||
""".format(
|
||||
email_numerator=numerateur.email.item(),
|
||||
today = today
|
||||
)
|
||||
drop_temp_table = """
|
||||
DROP TABLE IF EXISTS md.temp_import_personne;
|
||||
"""
|
||||
with con.begin() as cnx:
|
||||
cnx.execute(update_structure_name)
|
||||
cnx.execute(drop_temp_table)
|
||||
|
||||
@@ -0,0 +1,484 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-.
|
||||
|
||||
from sqlalchemy import create_engine #, text
|
||||
from sqlalchemy.engine import URL
|
||||
from shapely.geometry import Point
|
||||
from shapely.ops import nearest_points
|
||||
import pandas as pd
|
||||
import geopandas as gpd
|
||||
from pycen import con_sicen as con
|
||||
|
||||
|
||||
numerisateur = ['GEIER Colas']
|
||||
lib_etude = 'Echange de données (partenaires)'
|
||||
# précision possible :
|
||||
# GPS / 0 à 10m / 10 à 100m / 100 à 500m / commune / lieu-dit
|
||||
precision_gps = 'GPS'
|
||||
|
||||
dict_sp = {
|
||||
'Dilatata dilatata (Gould, 1841)':'Menetus dilatatus (A.A. Gould, 1841)',
|
||||
'Gomphocerippus biguttulus biguttulus (Linnaeus, 1758)':'Chorthippus biguttulus biguttulus (Linnaeus, 1758)',
|
||||
'Gomphocerippus mollis mollis (Charpentier, 1825)':'Chorthippus mollis mollis (Charpentier, 1825)',
|
||||
}
|
||||
dict_columns = {
|
||||
'TAXO_LATIN_C':'nom_latin',
|
||||
'RELV_NOM':'lot_donnees',
|
||||
'OBSE_DATE_J':'date_obs',
|
||||
'OBSE_LON':'longitude',
|
||||
'OBSE_LAT':'latitude',
|
||||
'SITE_NOM':'localisation',
|
||||
'OBSE_PLACE':'rmq_loc',
|
||||
'OBSV_LIBEL':'observateur',
|
||||
'DETM_LIBEL':'validateur',
|
||||
'OBSE_COMMENT':'remarque_obs',
|
||||
'OBSE_PCOLE_CHOI':'protocole',
|
||||
'OBSE_PRECIS_CHOI':'effectif_textuel',
|
||||
'OBSE_NOMBRE':'effectif',
|
||||
'OBSE_TIME':'heure_obs',
|
||||
'OBSE_COMP_CHOI':'reprostatut',
|
||||
'OBSE_CARACT_CHOI':'reprocaract',
|
||||
'OBSE_SIG_OBJ_ID':'id_origine',
|
||||
'OBSE_NEBUL_CHOI':'nebulosite',
|
||||
'OBSE_TPTAIR':'temperature',
|
||||
'OBSE_VENT_CHOI':'vent'
|
||||
}
|
||||
# dict_columns = {
|
||||
# 'Nom scientifique':'nom_latin',
|
||||
# 'CD_nom':'cd_nom',
|
||||
# 'Observateur':'observateur',
|
||||
# 'Date_Relevé':'date_obs',
|
||||
# 'Code_Relevé':'id_waypoint',
|
||||
# 'X lambert 93':'longitude',
|
||||
# 'Y Lambert 93':'latitude',
|
||||
# 'Altinf_Relevé':'elevation',
|
||||
# 'Nom_LieuDit':'localisation',
|
||||
# 'Description_Relevé':'remarque_obs',
|
||||
# 'Statuts':'remarque_obsStatut',
|
||||
# 'Lot_données':'lot_donnees',
|
||||
# 'Localisation_Relevé':'rmq_loc'
|
||||
# }
|
||||
|
||||
|
||||
|
||||
# sql = 'SELECT * FROM %s.%s LIMIT 10;' % ('saisie','saisie_observation')
|
||||
# saisie = gpd.read_postgis(
|
||||
# sql = sql,
|
||||
# con = con,
|
||||
# geom_col = 'geometrie'
|
||||
# )
|
||||
|
||||
def near(point,df2,geom_union,src_column):
|
||||
# find the nearest point and return the corresponding Place value
|
||||
geom2_col = df2.geometry.name
|
||||
nearest = df2[geom2_col] == nearest_points(point, geom_union)[1]
|
||||
# print(nearest)
|
||||
# print(df2[nearest][src_column])
|
||||
return df2[nearest][src_column].values[0]
|
||||
|
||||
|
||||
def get_refGeomTable(table, schema,geom_col='geometrie',colnames=None,mask=None,buffer=None):
|
||||
'''
|
||||
Récupération de données géométriques dasn une table de référence
|
||||
'''
|
||||
pkey = con.dialect.get_pk_constraint(con,table,schema)['constrained_columns'][0]
|
||||
if colnames:
|
||||
colnames = [pkey,*colnames]
|
||||
else:
|
||||
colnames = [pkey]
|
||||
cols = ','.join(colnames)
|
||||
sql = 'SELECT %s,%s FROM %s.%s' % (cols,geom_col,schema,table)
|
||||
if mask:
|
||||
if buffer :
|
||||
mask = mask.buffer(buffer)
|
||||
sql = sql + """ WHERE ST_Intersects({geom_col}, 'SRID=2154;{mask}')""".format(
|
||||
geom_col=geom_col,
|
||||
mask=mask)
|
||||
gdf = gpd.read_postgis(
|
||||
sql,con,
|
||||
geom_col=geom_col)
|
||||
return gdf
|
||||
|
||||
|
||||
def get_refTable(table, schema, colnames=None):
|
||||
pkey = con.dialect.get_pk_constraint(con,table,schema)['constrained_columns'][0]
|
||||
if colnames:
|
||||
colnames = [pkey,*colnames]
|
||||
else:
|
||||
colnames = [pkey]
|
||||
|
||||
sql = 'SELECT %s' % (','.join(colnames))
|
||||
sql = sql + ' FROM %s.%s' % (schema,table)
|
||||
df = pd.read_sql(sql,con)
|
||||
return df
|
||||
|
||||
|
||||
def create_geometry(x,y):
|
||||
geom = [Point(xy) for xy in zip(x, y)]
|
||||
return geom
|
||||
|
||||
|
||||
def normalize_obs(obs,name='first',sep=';'):
|
||||
'''
|
||||
:obs: pd.Series. Colonne des observateurs.
|
||||
:name: str. 'first' si le nom est cité en premier.
|
||||
'first',None. Défault 'first'.
|
||||
:sep: str. séparateur de personne dans le cas de
|
||||
multi-observateur. Défault ';'.
|
||||
'''
|
||||
obs = obs.replace(sep,',',regex=True)
|
||||
obs = obs.replace(' ,',',',regex=True)
|
||||
obs = obs.replace(', ',',',regex=True)
|
||||
obs = obs.str.split(',',expand=True).stack()
|
||||
obs = obs.str.split(' ',expand=True)
|
||||
if name == 'first':
|
||||
obs[0] = obs[0].str.upper()
|
||||
obs[1] = obs[1].str[0].str.upper() + obs[1].str[1:]
|
||||
obs['pers'] = obs[0] + ' ' + obs[1]
|
||||
else:
|
||||
obs[1] = obs[1].str.upper()
|
||||
obs[0] = obs[0].str[0].str.upper() + obs[0].str[1:]
|
||||
obs['pers'] = obs[1] + ' ' + obs[0]
|
||||
obs = obs['pers'].unstack()
|
||||
obs['pers'] = obs[0]
|
||||
if 1 in obs.columns:
|
||||
obs.loc[~obs[1].isna(),'pers'] = obs.loc[~obs[1].isna(),'pers'] + '&' +\
|
||||
obs.loc[~obs[1].isna(),1]
|
||||
return obs['pers']
|
||||
|
||||
|
||||
def get_pers(obs):
|
||||
'''
|
||||
:obs: pd.Series. Colonne des observateurs.
|
||||
'''
|
||||
obs = pd.Series(obs.unique())
|
||||
obs = obs.str.split(' ',expand=True)
|
||||
obs_name = "','".join(obs[0].str.upper().tolist())
|
||||
obs_surn = "','".join(obs[1].tolist())
|
||||
sql = '''
|
||||
SELECT
|
||||
concat(nom,' ',prenom) pers,
|
||||
id_personne,
|
||||
id_structure
|
||||
FROM md.personne
|
||||
WHERE nom in ('%s')
|
||||
AND prenom in ('%s')
|
||||
AND id_structure != 49
|
||||
;''' % (obs_name,obs_surn)
|
||||
obs = pd.read_sql(sql,con)
|
||||
return obs
|
||||
|
||||
|
||||
# import TAXREF
|
||||
taxref = pd.read_sql_table(
|
||||
table_name='taxref',
|
||||
con=con,
|
||||
schema='inpn'
|
||||
)
|
||||
taxref = taxref[[
|
||||
'regne','phylum','classe','ordre',
|
||||
'famille','nom_valide','nom_vern',
|
||||
'nom_complet','cd_nom']]
|
||||
|
||||
taxref['nom_latin'] = taxref.nom_complet.str.replace(' \(\#ref\)','',regex=True).copy()
|
||||
# taxref['nom_latin2'] = taxref.nom_latin.str.split(r' \(|, [0-9]',expand=True)[0]
|
||||
tmp = taxref[
|
||||
(taxref.nom_latin=='Euconulus Reinhardt, 1883')&
|
||||
(taxref.cd_nom!='192372')].index
|
||||
taxref.drop(tmp,inplace=True)
|
||||
|
||||
|
||||
# import DATA
|
||||
path = '/home/colas/Documents/9_PROJETS/4_SICEN/IMPORT/'
|
||||
file = 'Données Le Cheylas GD_2020.xlsx'
|
||||
# file = 'DonneesMeF.ods'
|
||||
df = pd.read_excel(path+file)
|
||||
# df = pd.read_csv(path+file)
|
||||
|
||||
# mise en forme DATA
|
||||
df.index.name = 'id'
|
||||
df.rename(columns=dict_columns, inplace=True)
|
||||
keep_col = df.columns[df.columns.isin(dict_columns.values())]
|
||||
df = df[keep_col]
|
||||
df.nom_latin.replace(dict_sp,inplace=True)
|
||||
df['statut_validation'] = 'validée'
|
||||
# df['lieu_dit'] = df['localisation'].copy()
|
||||
|
||||
if 'EFFC_OIS_Nb_Adult_Mâl' in df.columns:
|
||||
df.loc[~df.EFFC_OIS_Nb_Adult_Mâl.isna(),'type_effectif'] = 'Adulte'
|
||||
del df['EFFC_OIS_Nb_Adult_Mâl']
|
||||
if 'id_origine' in df.columns:
|
||||
df.loc[~df.id_origine.isna(),'id_origine'] = df.loc[~df.id_origine.isna(),'id_origine'].astype(int).astype(str)
|
||||
# Concaténation des remarques de localisation
|
||||
if 'rmq_loc' in df.columns:
|
||||
df.loc[~df.rmq_loc.isna(),'localisation'] = df.loc[~df.rmq_loc.isna(),'localisation'] + \
|
||||
', ' + df.loc[~df.rmq_loc.isna(),'rmq_loc']
|
||||
del df['rmq_loc']
|
||||
# Concaténation des remarques observateurs
|
||||
if 'remarque_obs' in df.columns:
|
||||
df.loc[~df.remarque_obs.isna(),'remarque_obs'] = 'Commentaires : ' + \
|
||||
df.loc[~df.remarque_obs.isna(),'remarque_obs']
|
||||
if 'remarque_obsStatut' in df.columns:
|
||||
df.loc[~df.remarque_obsStatut.isna(),'remarque_obs'] = df.loc[~df.remarque_obsStatut.isna(),'remarque_obs'] + \
|
||||
'; rmq_statut : ' + df.loc[~df.remarque_obsStatut.isna(),'remarque_obsStatut']
|
||||
del df['remarque_obsStatut']
|
||||
if 'nebulosite' in df.columns:
|
||||
df.loc[~df.nebulosite.isna(),'remarque_obs'] = df.loc[~df.nebulosite.isna(),'remarque_obs'] + \
|
||||
'; nebul : ' + df.loc[~df.nebulosite.isna(),'nebulosite']
|
||||
del df['nebulosite']
|
||||
if 'temperature' in df.columns:
|
||||
df.loc[~df.temperature.isna(),'remarque_obs'] = df.loc[~df.temperature.isna(),'remarque_obs'] + \
|
||||
'; temp : ' + df.loc[~df.temperature.isna(),'temperature'].astype(str)
|
||||
del df['temperature']
|
||||
if 'vent' in df.columns:
|
||||
df.loc[~df.vent.isna(),'remarque_obs'] = df.loc[~df.vent.isna(),'remarque_obs'] + \
|
||||
'; ' + df.loc[~df.vent.isna(),'vent']
|
||||
del df['vent']
|
||||
if 'id_waypoint' in df.columns:
|
||||
df.loc[~df.id_waypoint.isna(),'id_waypoint'] = df.loc[~df.id_waypoint.isna(),'id_waypoint']\
|
||||
.astype(int).astype(str)
|
||||
if 'precision' not in df.columns:
|
||||
df['precision'] = 'GPS'
|
||||
|
||||
# format date / time
|
||||
# df['date_obs'] = df['date_obs'].astype(str).astype(int)
|
||||
# df['date_obs'] = pd.to_datetime(df['date_obs'])
|
||||
# df['date_obs'].astype('datetime64')
|
||||
# pd.to_datetime(df['heure_obs'],format='%H:%M:%S')
|
||||
# df['heure_obs']
|
||||
|
||||
# create geometrie
|
||||
df['geometrie'] = create_geometry(df.longitude, df.latitude)
|
||||
if all(df.longitude < 10):
|
||||
epsg = 4326
|
||||
elif all(df.longitude > 900000):
|
||||
epsg = 2154
|
||||
df = df.set_geometry('geometrie', crs=epsg)
|
||||
if epsg == 4326:
|
||||
df.to_crs(2154, inplace=True)
|
||||
elif epsg == 2154:
|
||||
df['longitude'] = df.to_crs(4326).geometrie.x
|
||||
df['latitude'] = df.to_crs(4326).geometrie.y
|
||||
df_union = df.unary_union
|
||||
|
||||
# join code_INSEE & Lieu-dit
|
||||
# commune
|
||||
com = get_refGeomTable(
|
||||
'commune', 'ign_bd_topo',
|
||||
geom_col='geometrie',mask=df_union)
|
||||
com_union = com.unary_union
|
||||
# lieu_dit
|
||||
ldt = get_refGeomTable(
|
||||
'lieu_dit', 'ign_bd_topo', colnames=['nom'],
|
||||
geom_col='geometrie',mask=com_union, buffer=1000)
|
||||
ldt.rename(columns={'id': 'id_lieu_dit'}, inplace=True)
|
||||
ldt_union = ldt.unary_union
|
||||
# jointure lieu_dit / communes
|
||||
if 'lieu_dit' in df.columns:
|
||||
df['lieu_dit'] = df.lieu_dit.str.lower()
|
||||
df = pd.merge(df,ldt[['id_lieu_dit','nom']],left_on='lieu_dit',right_on='nom')
|
||||
del df['nom']
|
||||
del df['lieu_dit']
|
||||
else:
|
||||
df['id_lieu_dit'] = [
|
||||
near(geom,ldt,ldt_union,'id_lieu_dit')
|
||||
for geom in df.geometrie]
|
||||
df = gpd.sjoin(df,com)
|
||||
del df['index_right']
|
||||
|
||||
# get observateur/validateur
|
||||
df['observateur'] = normalize_obs(df['observateur'], sep='-')
|
||||
obs = get_pers(df.observateur)
|
||||
if not obs.pers.is_unique:
|
||||
raise Exception('duplicate observateur')
|
||||
ind = obs[(obs.pers == 'GRANGE Benjamin')&(obs.id_structure == 1)].index
|
||||
obs.drop(ind, inplace=True)
|
||||
|
||||
obs['id_personne'] = obs['id_personne'].astype(str)
|
||||
obs['id_structure'] = obs['id_structure'].astype(str)
|
||||
df['observateur'] = df['observateur'].replace(
|
||||
to_replace=[*obs.pers],
|
||||
value=[*obs.id_personne],
|
||||
regex=True)
|
||||
if 'validateur' in df.columns:
|
||||
val = get_pers(df.validateur)
|
||||
val['id_personne'] = val['id_personne'].astype(str)
|
||||
df['validateur'].replace(*obs.pers,*obs.id_personne, inplace=True)
|
||||
else:
|
||||
df['validateur'] = df['observateur'].str.split('&',expand=True)[0]
|
||||
for p in obs.iterrows():
|
||||
o = p[1]['id_personne']
|
||||
s = p[1]['id_structure']
|
||||
df.loc[df.observateur.str.contains(o),'structure'] = s
|
||||
|
||||
# get numerisateur
|
||||
num = get_pers(pd.Series(numerisateur))
|
||||
df['numerisateur'] = num.id_personne.astype(str)[0]
|
||||
|
||||
# get id_lot / id_etude
|
||||
lot = get_refTable('lot_donnee','md',colnames=['libelle'])
|
||||
etude = get_refTable('etude','md',colnames=['nom_etude'])
|
||||
df = pd.merge(df,lot,left_on='lot_donnees',right_on='libelle')
|
||||
del df['libelle']
|
||||
del df['lot_donnees']
|
||||
df['id_etude'] = etude.loc[etude.nom_etude == lib_etude,'id_etude'].values[0]
|
||||
|
||||
# get protocole
|
||||
if 'protocole' in df.columns:
|
||||
proto = get_refTable('protocole','md',colnames=['libelle'])
|
||||
df.protocole = df.protocole \
|
||||
.replace(
|
||||
to_replace=[
|
||||
'Capture à vue (à la main, au filet à papillon...)',
|
||||
"Recherche ciblée (bouses, cadavres d'animaux...)"],
|
||||
value=['Capture à vue',"Recherche d'indices de présence"])
|
||||
|
||||
df = pd.merge(df,proto, left_on='protocole',right_on='libelle')
|
||||
del df['libelle']
|
||||
del df['protocole']
|
||||
|
||||
|
||||
# join taxref
|
||||
if 'cd_nom' in df.columns:
|
||||
if df['cd_nom'].dtype == int: df['cd_nom'] = df['cd_nom'].astype(str)
|
||||
if 'nom_latin' in taxref.columns: del taxref['nom_latin']
|
||||
if 'nom_latin' in df.columns: del df['nom_latin']
|
||||
df = df.merge(taxref,on='cd_nom')
|
||||
else:
|
||||
df.nom_latin = df.nom_latin.str.strip()
|
||||
tt = taxref[taxref.nom_latin.isin(df.nom_latin.unique())]
|
||||
len_taxo = df.nom_latin.unique().shape[0]
|
||||
len_taxref = tt.shape[0]
|
||||
if len_taxo == len_taxref:
|
||||
df = df.merge(tt,on='nom_latin')
|
||||
del df['nom_latin']
|
||||
else:
|
||||
contain = '|'.join(df.nom_latin.unique())
|
||||
tt = taxref[taxref.nom_latin.str.contains(contain)]
|
||||
len_taxref = tt.shape[0]
|
||||
if len_taxo == len_taxref:
|
||||
df = df.merge(tt,on='nom_latin')
|
||||
del df['nom_latin']
|
||||
else:
|
||||
raise Exception('error corespondance cd_nom')
|
||||
contain = '|'.join(df.nom_latin.str.split(r' \(|, [0-9]',expand=True)[0].unique())
|
||||
tt = taxref[taxref.nom_latin2.str.contains(contain)]
|
||||
|
||||
|
||||
# get statuts de reproduction
|
||||
if 'reprostatut' in df.columns:
|
||||
if 'reprocaract' in df.columns:
|
||||
df.loc[(~df.reprostatut.isna())&(~df.reprocaract.isna()),'remarque_obs'] = df['remarque_obs'] + \
|
||||
'; rmq_repro : ' + df.loc[(~df.reprostatut.isna())&(~df.reprocaract.isna()),'reprocaract']
|
||||
df.loc[df.reprostatut.isna(),'reprostatut'] = df.loc[df.reprostatut.isna(),'reprocaract']
|
||||
del df['reprocaract']
|
||||
|
||||
sql = 'SELECT unnest(enum_range(null::{0}.{1})) {1}'.format('saisie','enum_reprostatut')
|
||||
reprostat = pd.read_sql(sql,con)
|
||||
reprostat = pd.concat(
|
||||
[reprostat,reprostat.enum_reprostatut.str.split('_',expand=True)],
|
||||
axis=1 )
|
||||
lst_phyl = df.phylum.unique()
|
||||
lst_clas = set([*df.classe.unique()])
|
||||
lst_ordr = set([*df.ordre.unique()])
|
||||
AMP = set(['Amphibia'])
|
||||
CHIR = set([])
|
||||
INV = set(['Bivalvia','Gastropoda'])
|
||||
MAM = set([])
|
||||
ODO = set(['Hexapoda'])
|
||||
ordr_ODO = set(['Odonata'])
|
||||
ordr_INV = set(['Orthoptera','Lepidoptera'])
|
||||
OIS = set(['Aves'])
|
||||
REP = set([])
|
||||
|
||||
if any(lst_clas.intersection(AMP)):
|
||||
# AMP = list(AMP)
|
||||
repro = reprostat[reprostat[0]=='AMP']
|
||||
df.loc[df.classe.isin(AMP),'reprostatut'] = df[df.classe.isin(AMP)].reprostatut \
|
||||
.replace(
|
||||
to_replace='Reproduction confirmée',
|
||||
value='Accouplement') \
|
||||
.replace(to_replace=[*repro[1]],value=[*repro.enum_reprostatut])
|
||||
# df.loc[df.classe.isin(AMP),'reprostatut'] = df[df.classe.isin(AMP)].reprostatut \
|
||||
# .replace(to_replace=[*repro[1]],value=[*repro.enum_reprostatut])
|
||||
|
||||
if any(lst_clas.intersection(CHIR)):
|
||||
# CHIR = list(CHIR)
|
||||
repro = reprostat[reprostat[0]=='CHIR']
|
||||
|
||||
if any(lst_clas.intersection(MAM)):
|
||||
# MAM = list(MAM)
|
||||
repro = reprostat[reprostat[0]=='MAM']
|
||||
|
||||
if any(lst_clas.intersection(INV)):
|
||||
# INV = list(INV)
|
||||
repro = reprostat[reprostat[0]=='INV']
|
||||
df.loc[df.classe.isin(INV),'reprostatut'] = df[df.classe.isin(INV)].reprostatut \
|
||||
.replace(
|
||||
to_replace='Reproduction confirmée',
|
||||
value='Accouplement') \
|
||||
.replace(to_replace=[*repro[1]],value=[*repro.enum_reprostatut])
|
||||
# df.loc[df.classe.isin(INV),'reprostatut'] = df[df.classe.isin(INV)].reprostatut \
|
||||
# .replace(to_replace=[*repro[1]],value=[*repro.enum_reprostatut])
|
||||
|
||||
if any(lst_clas.intersection(ODO)) and any(lst_ordr.intersection(ordr_INV)):
|
||||
# ODO = list(ODO)
|
||||
# ordr_INV = list(ordr_INV)
|
||||
repro = reprostat[reprostat[0]=='INV']
|
||||
df.loc[(df.classe.isin(ODO))&(df.ordre.isin(ordr_INV)),'reprostatut'] = \
|
||||
df[(df.classe.isin(ODO))&(df.ordre.isin(ordr_INV))].reprostatut \
|
||||
.replace(
|
||||
to_replace='Reproduction confirmée',
|
||||
value='Accouplement') \
|
||||
.replace(to_replace=[*repro[1]],value=[*repro.enum_reprostatut])
|
||||
# df.loc[(df.classe.isin(ODO))&(df.ordre.isin(ordr_INV)),'reprostatut'] = \
|
||||
# df[(df.classe.isin(ODO))&(df.ordre.isin(ordr_INV))].reprostatut \
|
||||
# .replace(to_replace=[*repro[1]],value=[*repro.enum_reprostatut])
|
||||
# ODO = set(ODO)
|
||||
|
||||
if any(lst_clas.intersection(ODO)) and any(lst_ordr.intersection(ordr_ODO)):
|
||||
# ODO = list(ODO)
|
||||
# ordr_ODO = list(ordr_ODO)
|
||||
repro = reprostat[reprostat[0]=='ODO']
|
||||
df.loc[(df.classe.isin(ODO))&(df.ordre.isin(ordr_ODO)),'reprostatut'] = \
|
||||
df[(df.classe.isin(ODO))&(df.ordre.isin(ordr_ODO))].reprostatut \
|
||||
.replace(
|
||||
to_replace=['Accouplement','Reproduction confirmée','Reproduction possible','Reproduction probable'],
|
||||
value=['Tandem','Exuvie/émergence','Mâles+Femelles','Immature']) \
|
||||
.replace(to_replace=[*repro[1]],value=[*repro.enum_reprostatut])
|
||||
# df.loc[(df.classe.isin(ODO))&(df.ordre.isin(ordr_ODO)),'reprostatut'] = \
|
||||
# df[(df.classe.isin(ODO))&(df.ordre.isin(ordr_ODO))].reprostatut \
|
||||
# .replace(to_replace=[*repro[1]],value=[*repro.enum_reprostatut])
|
||||
|
||||
if any(lst_clas.intersection(OIS)):
|
||||
# OIS = list(OIS)
|
||||
repro = reprostat[reprostat[0]=='OIS']
|
||||
df.loc[df.classe.isin(OIS),'reprostatut'] = df[df.classe.isin(OIS)].reprostatut \
|
||||
.replace(
|
||||
to_replace='Déplacement (marche, vol, nage)',
|
||||
value='Fuite - envol') \
|
||||
.replace(to_replace=[*repro[1]],value=[*repro.enum_reprostatut])
|
||||
# df.loc[df.classe.isin(OIS),'reprostatut'] = df[df.classe.isin(OIS)].reprostatut \
|
||||
# .replace(to_replace=[*repro[1]],value=[*repro.enum_reprostatut])
|
||||
|
||||
if any(lst_clas.intersection(REP)):
|
||||
# REP = list(REP)
|
||||
repro = reprostat[reprostat[0]=='REP']
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
df.to_postgis(
|
||||
name='saisie_observation',
|
||||
con=con,
|
||||
schema='saisie',
|
||||
if_exists='append',
|
||||
index=False,
|
||||
geom_col='geometrie',
|
||||
)
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
from pycen import con_sicen
|
||||
|
||||
def refresh_vm():
|
||||
sql = "REFRESH MATERIALIZED VIEW saisie.vm_synthese_observations WITH DATA;"
|
||||
with con_sicen.begin() as cnx:
|
||||
cnx.execute(sql)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
refresh_vm()
|
||||
@@ -0,0 +1,410 @@
|
||||
from os import replace
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from geoalchemy2 import Geometry
|
||||
# import numpy as np
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.engine import URL
|
||||
import geopandas as gpd
|
||||
from pycen import con_sicen as bckup_con
|
||||
|
||||
|
||||
# con = create_engine('postgresql+psycopg2://{1}:{0}@{2}:{3}/{4}'.format(pwd, user, adr, port, base), echo=False)
|
||||
# bckup_con = create_engine('postgresql+psycopg2://{1}:{0}@{2}:{3}/{4}'.format(bckup_pwd, bckup_user, bckup_adr, bckup_port, bckup_base), echo=False)
|
||||
|
||||
|
||||
def update_data(df, con, sch, tab, epsg=None):
|
||||
columns = df.columns.to_list()
|
||||
frame = df.copy()
|
||||
frame.replace("'","''", regex=True, inplace=True)
|
||||
pkey = con.dialect.get_pk_constraint(con, table_name=tab, schema=sch)['constrained_columns']
|
||||
for col in frame.select_dtypes(include=['object']).columns:
|
||||
test = frame[frame[col].str.contains("'", na=False)]
|
||||
if not test.empty:
|
||||
frame[col].replace("'", "''", inplace=True, regex=True)
|
||||
# print(frame[col])
|
||||
|
||||
|
||||
if 'geom' in columns or 'geometry' in columns:
|
||||
if epsg or df.crs:
|
||||
if not epsg:
|
||||
epsg = df.crs.to_epsg()
|
||||
name_geom = df.geometry.name
|
||||
frame[name_geom] = 'SRID={epsg};'.format(epsg=epsg) + df[name_geom].map(str)
|
||||
# else: return 'No crs define in update_data or in gdf'
|
||||
|
||||
for c, col in enumerate(columns):
|
||||
if c == 0:
|
||||
frame['insert'] = "('" + frame[col].map(str)
|
||||
# break
|
||||
else:
|
||||
frame['insert'] = frame['insert'] + "','" + frame[col].map(str)
|
||||
if c == len(columns)-1:
|
||||
frame['insert'] = frame['insert'] + "')"
|
||||
# if c == 0:
|
||||
# frame['insert'] = '("' + frame[col].map(str)
|
||||
# # break
|
||||
# else:
|
||||
# frame['insert'] = frame['insert'] + '","' + frame[col].map(str)
|
||||
# if c == len(columns)-1:
|
||||
# frame['insert'] = frame['insert'] + '")'
|
||||
|
||||
lst_cols = ', '.join(columns)
|
||||
lst_vals = ','.join(frame['insert'])
|
||||
lst_vals.replace("\'None\'", 'NULL', inplace=True)
|
||||
lst_dupKey = ', '.join([col + '=EXCLUDED.' + col for col in columns])
|
||||
lst_pkey = ','.join(pkey)
|
||||
|
||||
sql = '''INSERT INTO {sch}.{tab} ({lst_cols}) VALUES {lst_vals} ON CONFLICT ({lst_pkey}) DO UPDATE SET {lst_dupKey} ;'''.format(
|
||||
sch=sch, tab=tab, lst_cols=lst_cols, lst_vals=lst_vals, lst_dupKey=lst_dupKey, lst_pkey=lst_pkey)
|
||||
# sql = '''INSERT INTO {sch}.{tab} ({lst_cols})
|
||||
# VALUES {lst_vals}
|
||||
# ON CONFLICT DO NOTHING;
|
||||
# '''.format(sch=sch, tab=tab, lst_cols=lst_cols, lst_vals=lst_vals)
|
||||
try:
|
||||
con.execute(sql)
|
||||
# con.execute(text(sql))
|
||||
print('''
|
||||
Update OK !''')
|
||||
except Exception as exept:
|
||||
print(exept)
|
||||
|
||||
|
||||
df = pd.read_sql_table(
|
||||
table_name = 'personne',
|
||||
con = con,
|
||||
schema = 'md',
|
||||
index_col='id_personne')
|
||||
|
||||
tmp = pd.DataFrame(df.nom.str.split(' ',1).tolist(), columns = ['nom','prenom'], index=df.index)
|
||||
tmp = tmp[~tmp.prenom.isnull()]
|
||||
df.loc[tmp.index,'prenom'] = tmp['prenom']
|
||||
df.loc[tmp.index,'nom'] = tmp['nom']
|
||||
df['nom'] = [nom.replace('CEN38_','') for nom in df['nom']]
|
||||
df['nom'] = [nom.replace('GENTIANA_','') for nom in df['nom']]
|
||||
df['nom'] = [nom.replace('LPO38_','') for nom in df['nom']]
|
||||
df['nom'] = [nom.replace('LPO_','') for nom in df['nom']]
|
||||
df['nom'] = [nom.replace('CENRA_','') for nom in df['nom']]
|
||||
df['nom'] = [nom.replace('GRPLS_','') for nom in df['nom']]
|
||||
|
||||
rm_row = ['ISÈRE', 'Ecrin', 'FUR', 'BRIQUIR', 'BORGNE', '(FMBDS)', 'Isère', 'Rhône', '(Presonnel)', 'Monsieur', 'Batrachologique de France', '(Ecosphère)',
|
||||
'PIC VERT Association', 'BAGOUSSE', '(BIOTOPE)', '(Dauphinelle)', 'Cras', '(GRPLS)', 'et Vie Sociale', '(ONCFS)', 'campagne sauvetage amphibiens']
|
||||
df = df[~df.prenom.isin(rm_row)]
|
||||
tmp = pd.DataFrame(df.prenom.str.split(' ',1).tolist(), columns = ['nom','prenom'], index=df.index)
|
||||
tmp = tmp[~tmp.prenom.isnull()]
|
||||
tmp.drop(index=[100032,8628,8645,4238,8058,8070,8353,1099,1081,1323,1324], inplace=True)
|
||||
df.loc[tmp.index,'nom'] = df.loc[tmp.index,'nom'] + ' ' + tmp['nom']
|
||||
df.loc[tmp.index,'prenom'] = tmp['prenom']
|
||||
tmp = pd.DataFrame(
|
||||
df.loc[df.nom=='Abbé',['nom','prenom']].prenom.str.split(' ',1).tolist(),
|
||||
columns = ['prenom','nom'],
|
||||
index=df.loc[df.nom=='Abbé',['nom','prenom']].index)
|
||||
tmp2 = df.loc[df.nom=='Abbé',['nom','prenom']]
|
||||
df.loc[tmp.index,'nom'] = tmp['nom']
|
||||
df.loc[tmp.index,'prenom'] = tmp2['nom'] + ' ' + tmp['prenom']
|
||||
df.reset_index(inplace=True)
|
||||
tmp = df.copy()
|
||||
tmp = tmp[['id_personne','nom','prenom', 'id_structure']]
|
||||
# update_data(tmp,con,sch='md',tab='personne')
|
||||
|
||||
id_old = '8044'
|
||||
id_new = '1000014'
|
||||
sql='''SELECT * FROM saisie.saisie_observation WHERE observateur = '{0}' OR validateur = {0};'''.format(id_old)
|
||||
# sql = 'SELECT nom, prenom FROM md.personne WHERE id_personne= 6191;'
|
||||
pd.read_sql(
|
||||
sql=sql,
|
||||
con=con
|
||||
)[['observateur','validateur']]
|
||||
# df[['prenom', 'nom']].values.tolist()
|
||||
|
||||
|
||||
# sql = '''UPDATE saisie.saisie_observation SET validateur = 100077
|
||||
# WHERE validateur = 1110;'''
|
||||
try:
|
||||
con.execute(sql)
|
||||
# con.execute(text(sql))
|
||||
print('''
|
||||
Update OK !''')
|
||||
except Exception as exept:
|
||||
print(exept)
|
||||
|
||||
|
||||
tmpx = df.copy()
|
||||
tmpx = tmpx[['id_personne','nom','prenom','id_structure']]
|
||||
tutux = tmpx.drop_duplicates(['nom','prenom','id_structure'], keep=False)
|
||||
tomx = tmpx[~tmpx.id_personne.isin(tutux.id_personne)]
|
||||
tomx = tomx.sort_values('nom')
|
||||
|
||||
for nom in tomx.nom.unique().tolist():
|
||||
tmp1 = tomx[tomx.nom==nom]
|
||||
keep = max(tmp1.id_personne)
|
||||
rep = min(tmp1.id_personne)
|
||||
sql = '''UPDATE saisie.saisie_observation SET observateur = '{keep}' WHERE observateur = '{replace}';'''.format(
|
||||
keep=keep, replace=rep
|
||||
)
|
||||
# sql = '''UPDATE saisie.saisie_observation SET validateur = 100077
|
||||
# WHERE validateur = 1110;'''
|
||||
try:
|
||||
con.execute(sql)
|
||||
# con.execute(text(sql))
|
||||
print('''
|
||||
Update OK !''')
|
||||
except Exception as exept:
|
||||
print(exept)
|
||||
|
||||
sql='''SELECT * FROM saisie.saisie_observation WHERE observateur = '{replace}' OR validateur = {replace};'''.format(
|
||||
replace=rep
|
||||
)
|
||||
# sql = 'SELECT nom, prenom FROM md.personne WHERE id_personne= 6191;'
|
||||
test = pd.read_sql(
|
||||
sql=sql,
|
||||
con=con
|
||||
)[['observateur','validateur']]
|
||||
if test.empty:
|
||||
print(''' OK for %s'''%nom)
|
||||
|
||||
else:
|
||||
print('ERROR !!!!!!!!!')
|
||||
break
|
||||
|
||||
|
||||
print(nom)
|
||||
sql='''SELECT * FROM md.personne WHERE nom like '{nom}%';'''.format(nom=nom)
|
||||
tutu = pd.read_sql(
|
||||
sql=sql,
|
||||
con=con
|
||||
)
|
||||
if tutu.shape[0] > 1:
|
||||
df
|
||||
print('''
|
||||
{id_personne},{nom},{prenom},{id_structure}'''.format(
|
||||
id_personne=id_personne,nom=nom,prenom=prenom,id_structure=id_structure))
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.engine import URL
|
||||
from shapely.geometry import Point
|
||||
import matplotlib.pyplot as plt
|
||||
import contextily as cx
|
||||
import geopandas as gpd
|
||||
import pandas as pd
|
||||
from pycen import con_sicen as con
|
||||
|
||||
df1 = pd.read_sql_table(
|
||||
table_name = 'saisie_observation',
|
||||
con = con,
|
||||
schema = 'saisie',
|
||||
index_col='id_obs')
|
||||
|
||||
geometry = [Point(xy) for xy in zip(df1['longitude'], df['latitude'])]
|
||||
crs = 'epsg:4326'
|
||||
df = gpd.GeoDataFrame(df1, crs=crs, geometry=geometry)
|
||||
df = df.to_crs(epsg=3857)
|
||||
df_unkeep = df.drop_duplicates(keep=False)
|
||||
df_clean = df.drop_duplicates(keep='first')
|
||||
|
||||
df[~df.index.isin(df_clean.index)].sort_values('date_obs').index.values
|
||||
df[~df.index.isin(df_unkeep.index)].sort_values('date_obs').index.values
|
||||
|
||||
ax = df[~df.index.isin(df_unkeep.index)][['geometry']].plot()
|
||||
cx.add_basemap(ax)
|
||||
plt.show()
|
||||
|
||||
df.sort_index(inplace=True)
|
||||
data_drop = df[~df.index.isin(df.drop_duplicates(keep=False).index)]
|
||||
|
||||
tmp = [i.split('&') for i in df['observateur'].unique() ]
|
||||
lst_pers = [item for sublist in tmp for item in sublist]
|
||||
df_pers = pd.DataFrame(lst_pers, columns=['id_pers']).drop_duplicates()
|
||||
|
||||
sql='''SELECT * FROM md.personne WHERE id_personne IN ({ids});'''.format(ids=','.join(df_pers.id_pers))
|
||||
tutu = pd.read_sql(
|
||||
sql=sql,
|
||||
con=con
|
||||
)
|
||||
df_pers['id_pers'] = df_pers['id_pers'].astype(int)
|
||||
df_pers[~df_pers.id_pers.isin(tutu.id_personne)]
|
||||
|
||||
|
||||
|
||||
|
||||
tom = toto.copy()
|
||||
tom = tom[tom.observateur.str.contains('&')]
|
||||
tom['observateur'] = [idx.replace('1043','100086') for idx in tom['observateur']]
|
||||
tom['observateur'] = [idx.replace('6191','1000002') for idx in tom['observateur']]
|
||||
tom['observateur'] = [idx.replace('6051','1000013') for idx in tom['observateur']]
|
||||
tom['observateur'] = [idx.replace('1070','1000011') for idx in tom['observateur']]
|
||||
tom['observateur'] = [idx.replace('1088','100043') for idx in tom['observateur']]
|
||||
tom['observateur'] = [idx.replace('6045','1000004') for idx in tom['observateur']]
|
||||
tom['observateur'] = [idx.replace('6121','1000009') for idx in tom['observateur']]
|
||||
tom['observateur'] = [idx.replace('6355','1000003') for idx in tom['observateur']]
|
||||
tom['observateur'] = [idx.replace('8632','1000005') for idx in tom['observateur']]
|
||||
tom['observateur'] = [idx.replace('1034','100007') for idx in tom['observateur']]
|
||||
tom['observateur'] = [idx.replace('1110','100077') for idx in tom['observateur']]
|
||||
tom['observateur'] = [idx.replace('1075','1000022') for idx in tom['observateur']]
|
||||
tom['observateur'] = [idx.replace('1188','100038') for idx in tom['observateur']]
|
||||
tom['observateur'] = [idx.replace('8271','100017') for idx in tom['observateur']]
|
||||
tom['observateur'] = [idx.replace('1049','100053') for idx in tom['observateur']]
|
||||
tom['observateur'] = [idx.replace('8328','100034') for idx in tom['observateur']]
|
||||
tom['observateur'] = [idx.replace('1102','100062') for idx in tom['observateur']]
|
||||
tom['observateur'] = [idx.replace('1016','1000001') for idx in tom['observateur']]
|
||||
tom['observateur'] = [idx.replace('1187','100037') for idx in tom['observateur']]
|
||||
tom['observateur'] = [idx.replace('8018','1000008') for idx in tom['observateur']]
|
||||
tom['observateur'] = [idx.replace('4233','100067') for idx in tom['observateur']]
|
||||
tom['observateur'] = [idx.replace('8254','1000012') for idx in tom['observateur']]
|
||||
tom['observateur'] = [idx.replace('8015','100016') for idx in tom['observateur']]
|
||||
|
||||
df.sort_index(inplace=True)
|
||||
tom.sort_index(inplace=True)
|
||||
df.loc[df.index.isin(tom.index), 'observateur'] = tom['observateur']
|
||||
df.reset_index(inplace=True)
|
||||
tmp = df.copy()
|
||||
tmp = tmp[['id_obs','observateur']]
|
||||
update_data(tmp,con,sch='saisie',tab='saisie_observation')
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# Check user in suivi_saisie_observation existent tous!
|
||||
from datetime import datetime as dt
|
||||
df = pd.read_sql_table(
|
||||
table_name = 'suivi_saisie_observation',
|
||||
con = con,
|
||||
schema = 'saisie',
|
||||
)
|
||||
df.sort_values('date_operation', ascending=False, inplace=True)
|
||||
x = '2021-05-25 22:00:00'
|
||||
y = '2021-05-21 12:30:00'
|
||||
x = dt.strptime(x, '%Y-%m-%d %H:%M:%S')
|
||||
y = dt.strptime(y, '%Y-%m-%d %H:%M:%S')
|
||||
|
||||
ddf = df[(df.date_operation < x) & (df.date_operation > y)]
|
||||
ddf = ddf[['operation','utilisateur', 'id_obs', 'observateur', 'validateur', 'numerisateur']]
|
||||
|
||||
# Get all "id" observateur
|
||||
tmp_obs = [i.split('&') for i in ddf['observateur'].dropna().unique() ]
|
||||
lst_pers_obs = [item for sublist in tmp_obs for item in sublist]
|
||||
tmp_val = [i.split('&') for i in ddf['validateur'].dropna().unique().astype(int).astype(str) ]
|
||||
lst_pers_val = [item for sublist in tmp_val for item in sublist]
|
||||
tmp_num = [i.split('&') for i in ddf['numerisateur'].dropna().unique().astype(str) ]
|
||||
lst_pers_num = [item for sublist in tmp_num for item in sublist]
|
||||
lst_pers = lst_pers_obs + lst_pers_val + lst_pers_num
|
||||
df_pers = pd.DataFrame(lst_pers, columns=['id_pers']).drop_duplicates()
|
||||
|
||||
# Check if personnes exist
|
||||
pers_not_exist = []
|
||||
for i in df_pers.id_pers:
|
||||
sql = 'SELECT id_personne,prenom, nom, id_structure FROM md.personne WHERE id_personne = %s;'%i
|
||||
res = pd.read_sql(
|
||||
sql = sql,
|
||||
con = con
|
||||
)
|
||||
if res.empty:
|
||||
print(i)
|
||||
pers_not_exist += [i]
|
||||
|
||||
|
||||
sql = '''SELECT * FROM saisie.suivi_saisie_observation
|
||||
WHERE utilisateur = 'inconnu'
|
||||
AND date_operation < '2021-05-25 22:00:00'
|
||||
AND date_operation > '2021-05-21 12:30:00';'''
|
||||
pd.read_sql(
|
||||
sql=sql,
|
||||
con=con
|
||||
)
|
||||
sql = '''UPDATE saisie.suivi_saisie_observation SET utilisateur = 'colas.geier@cen-isere.org'
|
||||
WHERE utilisateur = 'inconnu'
|
||||
AND date_operation < '2021-05-25 22:00:00'
|
||||
AND date_operation > '2021-05-21 12:30:00';'''
|
||||
try:
|
||||
con.execute(sql)
|
||||
# con.execute(text(sql))
|
||||
print('''
|
||||
Update OK !''')
|
||||
except Exception as exept:
|
||||
print(exept)
|
||||
|
||||
|
||||
|
||||
rep = '8044'
|
||||
keep = '1000014'
|
||||
|
||||
sql_check ='''SELECT * FROM saisie.saisie_observation WHERE observateur like '%{replace}%' OR validateur = {replace};'''.format(
|
||||
replace=rep
|
||||
)
|
||||
sql_check ='''SELECT * FROM saisie.saisie_observation WHERE observateur = '{replace}' OR validateur = {replace};'''.format(
|
||||
replace=rep
|
||||
)
|
||||
sql_updt = '''
|
||||
UPDATE saisie.saisie_observation SET observateur = '{keep}' WHERE observateur = '{replace}';
|
||||
UPDATE saisie.saisie_observation SET numerisateur = {keep} WHERE numerisateur = {replace};
|
||||
'''.format(
|
||||
keep=keep, replace=rep
|
||||
)
|
||||
# sql = '''UPDATE saisie.saisie_observation SET validateur = 100077
|
||||
# WHERE validateur = 1110;'''
|
||||
|
||||
test = pd.read_sql(
|
||||
sql=sql_check,
|
||||
con=con
|
||||
)[['observateur','validateur']]
|
||||
if test.empty:
|
||||
print(''' OK for %s in table saisie'''%nom)
|
||||
else:
|
||||
try:
|
||||
con.execute(sql_updt)
|
||||
# con.execute(text(sql))
|
||||
print('''
|
||||
Update OK !''')
|
||||
except Exception as exept:
|
||||
print(exept)
|
||||
|
||||
sql_check2 ='''SELECT * FROM saisie.suivi_saisie_observation WHERE observateur = '{replace}' OR validateur = '{replace}';'''.format(
|
||||
replace=rep
|
||||
)
|
||||
sql_updt2 = '''
|
||||
UPDATE saisie.suivi_saisie_observation SET observateur = '{keep}' WHERE observateur = '{replace}';
|
||||
UPDATE saisie.suivi_saisie_observation SET numerisateur = {keep} WHERE numerisateur = {replace};
|
||||
'''.format(
|
||||
keep=keep, replace=rep
|
||||
)
|
||||
test2 = pd.read_sql(
|
||||
sql=sql_check2,
|
||||
con=con
|
||||
)[['observateur','validateur']]
|
||||
if test2.empty:
|
||||
print(''' OK for %s in table saisie'''%nom)
|
||||
else:
|
||||
try:
|
||||
con.execute(sql_updt2)
|
||||
# con.execute(text(sql))
|
||||
print('''
|
||||
Update OK !''')
|
||||
except Exception as exept:
|
||||
print(exept)
|
||||
|
||||
|
||||
|
||||
|
||||
sql='''SELECT * FROM saisie.saisie_observation WHERE observateur = '{replace}' OR validateur = {replace};'''.format(
|
||||
replace=rep
|
||||
)
|
||||
# sql = 'SELECT nom, prenom FROM md.personne WHERE id_personne= 6191;'
|
||||
test = pd.read_sql(
|
||||
sql=sql,
|
||||
con=con
|
||||
)[['observateur','validateur']]
|
||||
if test.empty:
|
||||
print(''' OK for %s'''%nom)
|
||||
|
||||
else:
|
||||
print('ERROR !!!!!!!!!')
|
||||
# break
|
||||
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-.
|
||||
|
||||
import geopandas as gpd
|
||||
import pandas as pd
|
||||
from geoalchemy2 import Geometry
|
||||
from sqlalchemy.engine import URL
|
||||
from sqlalchemy import create_engine
|
||||
import sys
|
||||
|
||||
|
||||
# Liste des sites
|
||||
lst_site = ['GOUT']
|
||||
|
||||
# Suppression des champs souhaités
|
||||
delete = True
|
||||
lst_cols_del = ['geometrie','date_debut_obs']
|
||||
|
||||
# Format d'extraction ['xls', 'xlsx', 'csv']
|
||||
FORMAT = 'xlsx'
|
||||
# Localistation du fichier de sortie
|
||||
OUTPUT = '~/Documents/tmp/Jean_Luc/'
|
||||
# Nom du fichier de sortie
|
||||
NAME_OUT = 'observation_site_GOUT'
|
||||
from pycen import con_sicen as engine_hom
|
||||
|
||||
# Parametres bdd HOMER (out)
|
||||
schema_hom = 'saisie'
|
||||
table_hom = 'saisie_observation'
|
||||
# engine_hom = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user_hom,pwd_hom,adr_hom,port_hom,base_hom), echo=False)
|
||||
|
||||
# Parametres bdd BART (in)
|
||||
# BART ETEINT : OBSOLETE
|
||||
adr_bart = '192.168.0.3'
|
||||
port_bart = '5432'
|
||||
base_bart = 'bd_cen38'
|
||||
schema_bart = 'sites'
|
||||
table_bart = "c_sites_zonages"
|
||||
url_bart = URL.create('postgresql+psycopg2',
|
||||
username=user_bart,
|
||||
password=pwd_bart,
|
||||
host=adr_bart,
|
||||
database=base_bart,
|
||||
)
|
||||
con_bart = create_engine(url_bart)
|
||||
# con_bart = create_engine('postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'.format(user_bart,pwd_bart,adr_bart,port_bart,base_bart), echo=False)
|
||||
|
||||
epsg = '2154'
|
||||
crs = 'EPSG:%s'%epsg
|
||||
|
||||
|
||||
|
||||
sql = "SELECT geom FROM {sch}.{tab} WHERE code_site in ('{lst_site}') AND type_zonage = 'ZO'".format(sch=schema_bart, tab=table_bart, lst_site="','".join(lst_site))
|
||||
gdf_site = gpd.read_postgis(
|
||||
sql = sql,
|
||||
con = con_bart,
|
||||
)
|
||||
if gdf_site.shape[0] > 1 :
|
||||
union = gdf_site.geom.cascaded_union
|
||||
Sunion = gpd.GeoSeries(union)
|
||||
gdf_union = gpd.GeoDataFrame(Sunion, geometry='geom', columns=['geom'], crs=crs)
|
||||
gdf_site = gdf_union
|
||||
|
||||
|
||||
sql = '''SELECT * FROM {sch}.{tab} WHERE ST_Intersects (geometrie, 'SRID={epsg};{poly}')'''.format(sch=schema_hom, tab=table_hom, epsg=epsg, poly=gdf_site.geom[0])
|
||||
gdf_saisie = gpd.read_postgis(
|
||||
sql = sql,
|
||||
con = engine_hom,
|
||||
geom_col = 'geometrie'
|
||||
)
|
||||
|
||||
df = pd.DataFrame(gdf_saisie)
|
||||
if delete:
|
||||
df.drop(columns= lst_cols_del, inplace=True)
|
||||
|
||||
if FORMAT in ['xls', 'xlsx']:
|
||||
df.to_excel(OUTPUT + NAME_OUT + '.' + FORMAT)
|
||||
if FORMAT == 'csv':
|
||||
df.to_file(OUTPUT + NAME_OUT + '.' + FORMAT)
|
||||
|
||||
sys.exit('END')
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding:UTF-8 -*-
|
||||
|
||||
from sqlalchemy import create_engine #, text
|
||||
from sqlalchemy.engine import URL
|
||||
import pandas as pd
|
||||
from pycen import con_sicen as con
|
||||
|
||||
|
||||
sql = '''
|
||||
SELECT * FROM inpn.taxons_isere_absents_taxref
|
||||
WHERE cd_nom::text NOT IN (SELECT cd_nom FROM inpn.taxref)'''
|
||||
taxabs = pd.read_sql_query(
|
||||
sql=sql,con=con
|
||||
)
|
||||
taxabs.cd_ref = taxabs.cd_ref.astype(int).astype(str)
|
||||
taxabs.drop(columns=['id_taxon','source'], inplace=True)
|
||||
taxabs.to_sql(
|
||||
name='taxref',
|
||||
con=con,
|
||||
schema='inpn',
|
||||
if_exists='append',
|
||||
index=False
|
||||
)
|
||||
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-.
|
||||
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.engine import URL
|
||||
from datetime import datetime as dt
|
||||
from pycen import update_to_sql
|
||||
import pandas as pd
|
||||
import geopandas as gpd
|
||||
from sqlalchemy.sql.expression import column
|
||||
|
||||
|
||||
nom_numerateur = 'GEIER'
|
||||
pnom_numerator = 'Colas'
|
||||
today = dt.now().date().isoformat()
|
||||
# Parametres bdd
|
||||
from pycen import con_sicen as con
|
||||
|
||||
# FONCTION
|
||||
def get_numerator(nom,pnom,id_structure=1):
|
||||
nom_numerateur = nom.upper()
|
||||
pnom_numerator = pnom[0].upper() + pnom[1:]
|
||||
query_get_numerator = """
|
||||
SELECT * FROM md.personne
|
||||
WHERE nom = '%s'
|
||||
AND prenom = '%s'
|
||||
AND id_structure = %i
|
||||
""" % (nom_numerateur,pnom_numerator,id_structure)
|
||||
numerateur = pd.read_sql_query(
|
||||
sql = query_get_numerator,
|
||||
con = con
|
||||
)
|
||||
return numerateur
|
||||
|
||||
# SELECT OBSERVATION
|
||||
sql = 'SELECT * FROM saisie.saisie_observation'
|
||||
saisie = gpd.read_postgis(
|
||||
sql,
|
||||
con,
|
||||
'geometrie')
|
||||
saisie.sort_values('id_obs', inplace=True)
|
||||
|
||||
# SELECT COMMUNE
|
||||
sql = 'SELECT * FROM ign_bd_topo.commune'
|
||||
com = gpd.read_postgis(
|
||||
sql,
|
||||
con,
|
||||
'geometrie')
|
||||
|
||||
|
||||
# JOINTURE spatiale
|
||||
df = gpd.sjoin(saisie[['id_obs','code_insee','geometrie']],com[['code_insee','geometrie']],op='within')
|
||||
df.sort_values('id_obs', inplace=True)
|
||||
tmp = df[~df.code_insee_left.eq(df.code_insee_right)].copy()
|
||||
tmp.rename(columns={'code_insee_right':'code_insee'}, inplace=True)
|
||||
# UPDATE SET new code_insee
|
||||
update_to_sql(
|
||||
tmp[['id_obs','code_insee']],
|
||||
con=con,
|
||||
table_name='saisie_observation',
|
||||
schema_name='saisie',
|
||||
key_name='id_obs',
|
||||
geom_col='geometrie'
|
||||
)
|
||||
|
||||
|
||||
# UPDATE utilisateur OF saisie.suivi_saisie_observation
|
||||
numerateur = get_numerator(nom_numerateur,pnom_numerator)
|
||||
update_structure_name = """
|
||||
UPDATE saisie.suivi_saisie_observation a
|
||||
SET utilisateur = REPLACE(utilisateur, 'inconnu', '{email_numerator}')
|
||||
WHERE date_operation > '{today}'
|
||||
;
|
||||
""".format(
|
||||
email_numerator=numerateur.email.item(),
|
||||
today = today
|
||||
)
|
||||
with con.begin() as cnx:
|
||||
cnx.execute(update_structure_name)
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.engine import URL
|
||||
from datetime import datetime as dt
|
||||
import pandas as pd
|
||||
import pycen
|
||||
|
||||
|
||||
|
||||
# Parametres bdd
|
||||
from pycen import con_sicen as con
|
||||
|
||||
sch = 'saisie'
|
||||
tab = 'saisie_observation'
|
||||
ids = 'id_obs'
|
||||
col_updt = 'phenologie'
|
||||
|
||||
sql = """
|
||||
SELECT
|
||||
{ids},
|
||||
{col}
|
||||
FROM {sch}.{tab}
|
||||
WHERE {col} = 'Inderterminé'
|
||||
""".format(sch=sch,tab=tab,ids=ids,col=col_updt)
|
||||
df = pd.read_sql_query(sql,con)
|
||||
df[col_updt] = 'Indéterminé'
|
||||
pycen.update_to_sql(df,con,tab,sch,ids,geom_col=None)
|
||||
|
||||
|
||||
tab_suivi = 'suivi_saisie_observation'
|
||||
ids_suivi = ['operation','date_operation','id_obs']
|
||||
col_updtSuivi = 'utilisateur'
|
||||
date = dt.today().date().isoformat()
|
||||
sql = """
|
||||
SELECT
|
||||
{ids},
|
||||
{col}
|
||||
FROM {sch}.{tab}
|
||||
WHERE {col} = 'inconnu'
|
||||
AND date_operation > '{date}'
|
||||
""".format(sch=sch,tab=tab_suivi,ids=','.join(ids_suivi),col=col_updtSuivi, date=date)
|
||||
df = pd.read_sql_query(sql,con)
|
||||
df[col_updtSuivi] = 'colas.geier@cen-isere.org'
|
||||
pycen.update_to_sql(
|
||||
df,
|
||||
con,
|
||||
table_name=tab_suivi,
|
||||
schema_name=sch,
|
||||
key_name = ids_suivi,
|
||||
geom_col=None)
|
||||
@@ -0,0 +1,43 @@
|
||||
from pycen import con_sicen as con
|
||||
import pandas as pd
|
||||
from datetime import datetime as dt
|
||||
|
||||
# FONCTION
|
||||
def get_numerator(nom,pnom,id_structure=1):
|
||||
nom_numerateur = nom.upper()
|
||||
pnom_numerator = pnom[0].upper() + pnom[1:]
|
||||
query_get_numerator = """
|
||||
SELECT * FROM md.personne
|
||||
WHERE nom = '%s'
|
||||
AND prenom = '%s'
|
||||
AND id_structure = %i
|
||||
""" % (nom_numerateur,pnom_numerator,id_structure)
|
||||
numerateur = pd.read_sql_query(
|
||||
sql = query_get_numerator,
|
||||
con = con
|
||||
)
|
||||
return numerateur
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
nom_numerateur = 'GEIER'
|
||||
pnom_numerator = 'Colas'
|
||||
today = dt.now().date().isoformat()
|
||||
|
||||
# UPDATE utilisateur OF saisie.suivi_saisie_observation
|
||||
numerateur = get_numerator(nom_numerateur,pnom_numerator)
|
||||
|
||||
update_structure_name = """
|
||||
UPDATE saisie.suivi_saisie_observation a
|
||||
SET utilisateur = REPLACE(utilisateur, 'inconnu', '{email_numerator}')
|
||||
WHERE date_operation > '{today}'
|
||||
;
|
||||
""".format(
|
||||
email_numerator=numerateur.email.item(),
|
||||
today = today
|
||||
)
|
||||
|
||||
with con.begin() as cnx:
|
||||
cnx.execute(update_structure_name)
|
||||
print('END UPDATE utilisateur OF saisie.suivi_saisie_observation')
|
||||
@@ -0,0 +1,273 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
|
||||
import geopandas as gpd
|
||||
from pycen import con_sicen
|
||||
|
||||
drop_v_saisie_ = "DROP VIEW IF EXISTS saisie.v_saisie_observation"
|
||||
v_saisie_observation = """
|
||||
CREATE OR REPLACE VIEW saisie.v_saisie_observation
|
||||
AS WITH auteur AS (
|
||||
SELECT p_1.id_personne,
|
||||
CASE
|
||||
WHEN p_1.prenom ~~ '(%%'::text THEN p_1.nom
|
||||
ELSE COALESCE((p_1.nom || ' '::text) || p_1.prenom)
|
||||
END AS personne,
|
||||
s_1.nom_structure AS structure
|
||||
FROM md.personne p_1
|
||||
JOIN md.structure s_1 USING (id_structure)
|
||||
)
|
||||
SELECT s.id_obs,
|
||||
s.date_obs,
|
||||
s.date_debut_obs,
|
||||
s.date_fin_obs,
|
||||
s.date_textuelle,
|
||||
s.regne,
|
||||
s.nom_vern,
|
||||
s.nom_complet,
|
||||
s.cd_nom,
|
||||
t.cd_ref,
|
||||
s.effectif_textuel,
|
||||
s.effectif_min,
|
||||
s.effectif_max,
|
||||
CASE
|
||||
WHEN s.regne = 'Plantae'::text THEN s.type_effectif
|
||||
ELSE NULL::text
|
||||
END AS strate_flore,
|
||||
CASE
|
||||
WHEN s.regne = 'Plantae'::text THEN s.phenologie
|
||||
ELSE NULL::text
|
||||
END AS phenologie_flore,
|
||||
CASE
|
||||
WHEN s.regne = 'Animalia'::text THEN s.type_effectif
|
||||
ELSE NULL::text
|
||||
END AS age_faune,
|
||||
CASE
|
||||
WHEN s.regne = 'Animalia'::text THEN s.phenologie
|
||||
ELSE NULL::text
|
||||
END AS sexe_faune,
|
||||
s.id_waypoint,
|
||||
s.longitude,
|
||||
s.latitude,
|
||||
s.localisation,
|
||||
md.liste_nom_auteur(s.observateur::character varying) AS observateur,
|
||||
a1.personne AS numerisateur,
|
||||
a2.personne AS validateur,
|
||||
md.liste_nom_structure(s.structure::character varying) AS structure,
|
||||
s.remarque_obs,
|
||||
s.code_insee,
|
||||
s.id_lieu_dit,
|
||||
s.diffusable,
|
||||
s."precision",
|
||||
s.statut_validation,
|
||||
e.nom_etude AS etude,
|
||||
p.libelle AS protocole,
|
||||
s.effectif,
|
||||
s.url_photo,
|
||||
s.commentaire_photo,
|
||||
s.decision_validation,
|
||||
s.heure_obs,
|
||||
s.determination,
|
||||
s.elevation,
|
||||
s.geometrie,
|
||||
s.phylum,
|
||||
s.classe,
|
||||
s.ordre,
|
||||
s.famille,
|
||||
s.nom_valide,
|
||||
s.qualification,
|
||||
s.reprostatut AS reprostatut_faune,
|
||||
s.obs_null,
|
||||
s.uuid,
|
||||
ld.libelle AS lot_donnee,
|
||||
s.id_origine
|
||||
FROM saisie.saisie_observation s
|
||||
LEFT JOIN auteur a1 ON a1.id_personne = s.numerisateur
|
||||
LEFT JOIN auteur a2 ON a2.id_personne = s.validateur
|
||||
LEFT JOIN md.etude e ON s.id_etude = e.id_etude
|
||||
LEFT JOIN md.protocole p ON s.id_protocole = p.id_protocole
|
||||
LEFT JOIN md.lot_donnee ld ON s.id_lot = ld.id_lot
|
||||
LEFT JOIN inpn.taxref t using (cd_nom)
|
||||
;
|
||||
"""
|
||||
|
||||
grant_v_saisie = """
|
||||
ALTER TABLE saisie.v_saisie_observation OWNER TO cgeier;
|
||||
GRANT ALL ON TABLE saisie.v_saisie_observation TO cgeier;
|
||||
"""
|
||||
|
||||
with con_sicen.begin() as cnx:
|
||||
cnx.execute(drop_v_saisie_)
|
||||
cnx.execute(v_saisie_observation)
|
||||
cnx.execute(grant_v_saisie)
|
||||
|
||||
|
||||
drop_vm_synthese = "DROP MATERIALIZED VIEW IF EXISTS saisie.vm_synthese_observations CASCADE;"
|
||||
vm_synthese_observations = """
|
||||
-- saisie.vm_synthese_observations source
|
||||
|
||||
CREATE MATERIALIZED VIEW saisie.vm_synthese_observations
|
||||
TABLESPACE pg_default
|
||||
AS WITH
|
||||
observateurs AS (
|
||||
SELECT personne.id_personne,
|
||||
(personne.nom || ' '::text) || personne.prenom AS nom_complet,
|
||||
personne.role,
|
||||
personne.id_structure
|
||||
FROM md.personne
|
||||
),
|
||||
structures AS (
|
||||
SELECT structure.id_structure,
|
||||
structure.nom_structure,
|
||||
structure.diffusable
|
||||
FROM md.structure
|
||||
)
|
||||
SELECT DISTINCT s.id_obs,
|
||||
e.nom_etude AS etude,
|
||||
p.libelle AS protocole,
|
||||
ld.libelle AS lot_donnee,
|
||||
s.date_obs,
|
||||
s.date_debut_obs,
|
||||
s.date_fin_obs,
|
||||
s.heure_obs,
|
||||
CASE
|
||||
WHEN tx.regne IS NOT NULL THEN tx.regne
|
||||
ELSE ta.regne
|
||||
END AS regne,
|
||||
--tx.phylum,
|
||||
CASE
|
||||
WHEN tx.phylum IS NOT NULL THEN tx.phylum
|
||||
ELSE ta.phylum
|
||||
END AS phylum,
|
||||
--tx.classe,
|
||||
CASE
|
||||
WHEN tx.classe IS NOT NULL THEN tx.classe
|
||||
ELSE ta.classe
|
||||
END AS classe,
|
||||
--tx.ordre,
|
||||
CASE
|
||||
WHEN tx.ordre IS NOT NULL THEN tx.ordre
|
||||
ELSE ta.ordre
|
||||
END AS ordre,
|
||||
tx.famille,
|
||||
CASE
|
||||
WHEN tx.group1_inpn IS NOT NULL THEN tx.group1_inpn
|
||||
ELSE ta.group1_inpn
|
||||
END AS group1_inpn,
|
||||
CASE
|
||||
WHEN tx.group2_inpn IS NOT NULL THEN tx.group2_inpn
|
||||
ELSE ta.group2_inpn
|
||||
END AS group2_inpn,
|
||||
CASE
|
||||
WHEN tx.lb_nom IS NOT NULL THEN tx.lb_nom
|
||||
ELSE ta.lb_nom
|
||||
END AS nom_latin,
|
||||
CASE
|
||||
WHEN tx.nom_vern IS NOT NULL THEN tx.nom_vern
|
||||
ELSE ta.lb_nom
|
||||
END AS nom_vern,
|
||||
CASE
|
||||
WHEN tx.nom_complet IS NOT NULL THEN tx.nom_complet
|
||||
ELSE ta.nom_complet
|
||||
END AS nom_complet,
|
||||
CASE
|
||||
WHEN tx.cd_nom IS NOT NULL THEN tx.cd_nom
|
||||
ELSE ta.cd_nom
|
||||
END AS cd_nom,
|
||||
CASE
|
||||
WHEN tx.cd_ref IS NOT NULL THEN tx.cd_ref
|
||||
ELSE ta.cd_ref
|
||||
END AS cd_ref,
|
||||
--s.cd_nom,
|
||||
--t.cd_ref,
|
||||
CASE
|
||||
WHEN s.obs_null IS TRUE THEN 'oui'::text
|
||||
ELSE 'non'::text
|
||||
END::character(3) AS absence_observation,
|
||||
s.effectif,
|
||||
s.effectif_min,
|
||||
s.effectif_max,
|
||||
s.effectif_textuel,
|
||||
CASE
|
||||
WHEN s.regne ~~ 'Plantae'::text THEN s.type_effectif
|
||||
ELSE NULL::text
|
||||
END::saisie.enum_type_effectif AS strate_flore,
|
||||
CASE
|
||||
WHEN s.regne ~~ 'Plantae'::text THEN s.phenologie
|
||||
ELSE NULL::text
|
||||
END::saisie.enum_phenologie AS phenologie_flore,
|
||||
CASE
|
||||
WHEN s.regne ~~ 'Animalia'::text THEN s.type_effectif
|
||||
ELSE NULL::text
|
||||
END::saisie.enum_age AS age_faune,
|
||||
CASE
|
||||
WHEN s.regne ~~ 'Animalia'::text THEN s.phenologie
|
||||
ELSE NULL::text
|
||||
END::saisie.enum_sexe AS sexe_faune,
|
||||
s.reprostatut AS reprostatut_faune,
|
||||
s.determination,
|
||||
s.remarque_obs AS rmq_observation,
|
||||
md.liste_nom_auteur(s.observateur::character varying) AS observateurs,
|
||||
md.liste_nom_auteur_structure(s.observateur::character varying) AS observateurs_v2,
|
||||
num.nom_complet AS numerisateur,
|
||||
md.liste_nom_structure(s.structure::character varying) AS structures,
|
||||
CASE
|
||||
WHEN s.diffusable IS TRUE THEN 'oui'::text
|
||||
ELSE 'non'::text
|
||||
END::character(3) AS diffusable,
|
||||
s.statut_validation,
|
||||
val.nom_complet AS validateur,
|
||||
s.decision_validation,
|
||||
s.code_insee AS insee_commune,
|
||||
c.nom AS nom_commune,
|
||||
l.nom AS lieu_dit,
|
||||
s.elevation AS altitude_z,
|
||||
s.longitude AS longitude_x,
|
||||
s.latitude AS latitude_y,
|
||||
s."precision",
|
||||
s.localisation AS rmq_localisation,
|
||||
s.id_origine,
|
||||
s.geometrie::geometry(Point,2154) AS geom
|
||||
FROM saisie.saisie_observation s
|
||||
LEFT JOIN inpn.taxref t ON s.cd_nom = t.cd_nom
|
||||
LEFT JOIN inpn.taxons_isere_absents_taxref ta ON s.cd_nom = ta.cd_nom::text
|
||||
LEFT JOIN ref_inpn_taxref.taxref_v11 tx ON s.cd_nom = tx.cd_nom::text
|
||||
JOIN md.etude e ON s.id_etude = e.id_etude
|
||||
LEFT JOIN md.protocole p ON s.id_protocole = p.id_protocole
|
||||
LEFT JOIN md.lot_donnee ld ON s.id_lot = ld.id_lot
|
||||
LEFT JOIN observateurs num ON s.numerisateur = num.id_personne
|
||||
LEFT JOIN observateurs val ON s.validateur = val.id_personne
|
||||
LEFT JOIN ign_bd_topo.commune c ON s.code_insee = c.code_insee
|
||||
LEFT JOIN ign_bd_topo.lieu_dit l ON s.id_lieu_dit = l.id
|
||||
WITH DATA;
|
||||
|
||||
-- View indexes:
|
||||
CREATE INDEX idx_vm_synthese_observations_date_debut_obs ON saisie.vm_synthese_observations USING btree (date_debut_obs);
|
||||
CREATE INDEX idx_vm_synthese_observations_date_fin_obs ON saisie.vm_synthese_observations USING btree (date_fin_obs);
|
||||
CREATE INDEX idx_vm_synthese_observations_date_obs ON saisie.vm_synthese_observations USING btree (date_obs);
|
||||
CREATE INDEX idx_vm_synthese_observations_group1_inpn ON saisie.vm_synthese_observations USING btree (group1_inpn);
|
||||
CREATE INDEX idx_vm_synthese_observations_group2_inpn ON saisie.vm_synthese_observations USING btree (group2_inpn);
|
||||
CREATE UNIQUE INDEX idx_vm_synthese_observations_id_obs ON saisie.vm_synthese_observations USING btree (id_obs);
|
||||
CREATE INDEX idx_vm_synthese_observations_nom_commune ON saisie.vm_synthese_observations USING btree (nom_commune);
|
||||
CREATE INDEX idx_vm_synthese_observations_nom_latin ON saisie.vm_synthese_observations USING btree (nom_latin);
|
||||
CREATE INDEX idx_vm_synthese_observations_observateurs ON saisie.vm_synthese_observations USING btree (observateurs);
|
||||
CREATE INDEX idx_vm_synthese_observations_structures ON saisie.vm_synthese_observations USING btree (structures);
|
||||
CREATE INDEX sidx_vm_synthese_observations_geom ON saisie.vm_synthese_observations USING gist (geom);
|
||||
|
||||
|
||||
COMMENT ON MATERIALIZED VIEW saisie.vm_synthese_observations IS 'Vue matérialisée de synthèse des données d''observations issuent de la table "saisie.saise_observation" et reformattée pour faciliter son utilisation dans QGIS. La vue est rafraîchie toutes les heures avec un script PSQL executer en SHELL via une tâche planifiée CRON côté serveur. Cette vue matérialisée peut également être rafraîchi manuellement avec la commande : REFRESH MATERIALIZED VIEW CONCURRENTLY';
|
||||
"""
|
||||
|
||||
grant_vm_synthese = """
|
||||
-- Permissions
|
||||
|
||||
ALTER TABLE saisie.vm_synthese_observations OWNER TO admin_ra;
|
||||
GRANT ALL ON TABLE saisie.vm_synthese_observations TO admin_ra;
|
||||
GRANT SELECT ON TABLE saisie.vm_synthese_observations TO grp_consult;
|
||||
GRANT ALL ON TABLE saisie.vm_synthese_observations TO grp_admin;
|
||||
GRANT SELECT ON TABLE saisie.vm_synthese_observations TO sicen2_gr_consult;
|
||||
"""
|
||||
with con_sicen.begin() as cnx:
|
||||
cnx.execute(drop_vm_synthese)
|
||||
cnx.execute(vm_synthese_observations)
|
||||
cnx.execute(grant_vm_synthese)
|
||||
@@ -0,0 +1,250 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
|
||||
import geopandas as gpd
|
||||
from pycen import con_sicen
|
||||
|
||||
drop_v_inv = "DROP VIEW IF EXISTS saisie.v_synthese_invertebre"
|
||||
v_synthese_invertebre = """
|
||||
CREATE OR REPLACE VIEW saisie.v_synthese_invertebre
|
||||
AS WITH update_data AS (
|
||||
SELECT
|
||||
vm.id_obs,
|
||||
MAX(s.date_operation) date_update
|
||||
FROM saisie.vm_synthese_observations vm
|
||||
LEFT JOIN saisie.suivi_saisie_observation s USING (id_obs)
|
||||
GROUP BY vm.id_obs
|
||||
)
|
||||
SELECT
|
||||
vm.id_obs,
|
||||
vm.etude,
|
||||
vm.protocole,
|
||||
vm.lot_donnee,
|
||||
s.date_operation,
|
||||
ud.date_update,
|
||||
vm.date_obs::date,
|
||||
vm.date_debut_obs::date,
|
||||
vm.date_fin_obs::date,
|
||||
vm.heure_obs::time,
|
||||
vm.regne,
|
||||
vm.classe,
|
||||
vm.ordre,
|
||||
vm.famille,
|
||||
vm.group1_inpn,
|
||||
vm.group2_inpn,
|
||||
vm.nom_latin,
|
||||
vm.nom_vern,
|
||||
vm.nom_complet,
|
||||
vm.cd_nom,
|
||||
vm.cd_ref,
|
||||
vm.absence_observation,
|
||||
vm.effectif,
|
||||
vm.effectif_min,
|
||||
vm.effectif_max,
|
||||
vm.effectif_textuel,
|
||||
vm.strate_flore,
|
||||
vm.phenologie_flore,
|
||||
vm.age_faune,
|
||||
vm.sexe_faune,
|
||||
vm.reprostatut_faune,
|
||||
vm.determination,
|
||||
vm.rmq_observation,
|
||||
vm.observateurs,
|
||||
vm.observateurs_v2,
|
||||
vm.numerisateur,
|
||||
vm.structures,
|
||||
vm.diffusable,
|
||||
vm.statut_validation,
|
||||
vm.validateur,
|
||||
vm.decision_validation,
|
||||
vm.insee_commune,
|
||||
vm.nom_commune,
|
||||
vm.lieu_dit,
|
||||
vm.altitude_z,
|
||||
vm.longitude_x,
|
||||
vm.latitude_y,
|
||||
vm."precision",
|
||||
vm.rmq_localisation,
|
||||
vm.geom,
|
||||
so.id_origine
|
||||
FROM saisie.vm_synthese_observations vm
|
||||
LEFT JOIN ref_inpn_taxref.taxref_v11 tax ON vm.cd_nom::integer = tax.cd_nom
|
||||
JOIN saisie.saisie_observation so USING (id_obs)
|
||||
LEFT JOIN update_data ud USING (id_obs)
|
||||
LEFT JOIN (SELECT * FROM saisie.suivi_saisie_observation WHERE operation = 'INSERT'::text) s USING (id_obs)
|
||||
WHERE tax.regne = 'Animalia'::text AND tax.phylum <> 'Chordata'::text OR (vm.cd_nom::text = ANY (ARRAY['9999024'::text, '9999025'::text, '9999032'::text, '9999055'::text, '9999061'::text, '9999062'::text, '9999069'::text, '9999036'::text, '9999040'::text, '9999072'::text, '9999045'::text, '9999047'::text, '9999051'::text, '9999052'::text, '9999053'::text, '9999081'::text, '9999026'::text, '9999050'::text, '9999048'::text, '9999037'::text, '9999066'::text, '9999065'::text, '9999080'::text]));
|
||||
"""
|
||||
grant_v_inv = """
|
||||
ALTER TABLE saisie.v_synthese_invertebre OWNER TO cen_admin;
|
||||
GRANT ALL ON TABLE saisie.v_synthese_invertebre TO cen_admin;
|
||||
GRANT SELECT ON TABLE saisie.v_synthese_invertebre TO grp_consult;
|
||||
GRANT ALL ON TABLE saisie.v_synthese_invertebre TO grp_admin;
|
||||
"""
|
||||
with con_sicen.begin() as cnx:
|
||||
cnx.execute(drop_v_inv)
|
||||
cnx.execute(v_synthese_invertebre)
|
||||
cnx.execute(grant_v_inv)
|
||||
|
||||
|
||||
drop_v_vert = "DROP VIEW IF EXISTS saisie.v_synthese_vertebre"
|
||||
v_synthese_vertebre = """
|
||||
CREATE OR REPLACE VIEW saisie.v_synthese_vertebre
|
||||
AS WITH update_data AS (
|
||||
SELECT
|
||||
vm.id_obs,
|
||||
MAX(s.date_operation) date_update
|
||||
FROM saisie.vm_synthese_observations vm
|
||||
LEFT JOIN saisie.suivi_saisie_observation s USING (id_obs)
|
||||
GROUP BY vm.id_obs
|
||||
)
|
||||
SELECT
|
||||
vm.id_obs,
|
||||
vm.etude,
|
||||
vm.protocole,
|
||||
vm.lot_donnee,
|
||||
s.date_operation,
|
||||
ud.date_update,
|
||||
vm.date_obs::date,
|
||||
vm.date_debut_obs::date,
|
||||
vm.date_fin_obs::date,
|
||||
vm.heure_obs::time,
|
||||
vm.regne,
|
||||
vm.classe,
|
||||
vm.ordre,
|
||||
vm.famille,
|
||||
vm.group1_inpn,
|
||||
vm.group2_inpn,
|
||||
vm.nom_latin,
|
||||
vm.nom_vern,
|
||||
vm.nom_complet,
|
||||
vm.cd_nom,
|
||||
vm.cd_ref,
|
||||
vm.absence_observation,
|
||||
vm.effectif,
|
||||
vm.effectif_min,
|
||||
vm.effectif_max,
|
||||
vm.effectif_textuel,
|
||||
vm.strate_flore,
|
||||
vm.phenologie_flore,
|
||||
vm.age_faune,
|
||||
vm.sexe_faune,
|
||||
vm.reprostatut_faune,
|
||||
vm.determination,
|
||||
vm.rmq_observation,
|
||||
vm.observateurs,
|
||||
vm.observateurs_v2,
|
||||
vm.numerisateur,
|
||||
vm.structures,
|
||||
vm.diffusable,
|
||||
vm.statut_validation,
|
||||
vm.validateur,
|
||||
vm.decision_validation,
|
||||
vm.insee_commune,
|
||||
vm.nom_commune,
|
||||
vm.lieu_dit,
|
||||
vm.altitude_z,
|
||||
vm.longitude_x,
|
||||
vm.latitude_y,
|
||||
vm."precision",
|
||||
vm.rmq_localisation,
|
||||
vm.geom,
|
||||
so.id_origine
|
||||
FROM saisie.vm_synthese_observations vm
|
||||
LEFT JOIN ref_inpn_taxref.taxref_v11 tax ON vm.cd_nom::integer = tax.cd_nom
|
||||
JOIN saisie.saisie_observation so USING (id_obs)
|
||||
LEFT JOIN update_data ud USING (id_obs)
|
||||
LEFT JOIN (SELECT * FROM saisie.suivi_saisie_observation WHERE operation = 'INSERT'::text) s USING (id_obs)
|
||||
WHERE tax.regne = 'Animalia'::text AND tax.phylum = 'Chordata'::text OR (vm.cd_nom::text = ANY (ARRAY['9999042'::text, '9999028'::text, '9999029'::text, '9999056'::text, '9999058'::text, '9999067'::text, '9999068'::text, '9999030'::text, '9999031'::text, '9999034'::text, '9999034'::text, '9999035'::text, '9999035'::text, '9999038'::text, '9999039'::text, '9999070'::text, '9999073'::text, '9999057'::text, '9999054'::text, '9999049'::text, '9999022'::text, '9999027'::text, '9999043'::text, '9999044'::text, '9999046'::text, '9999041'::text, '9999033'::text, '9999071'::text, '9999064'::text, '9999063'::text, '9999060'::text, '9999059'::text, '9999074'::text, '9999023'::text, '9999082'::text, '9999083'::text]));
|
||||
"""
|
||||
grant_v_vert = """
|
||||
ALTER TABLE saisie.v_synthese_vertebre OWNER TO cen_admin;
|
||||
GRANT ALL ON TABLE saisie.v_synthese_vertebre TO cen_admin;
|
||||
GRANT SELECT ON TABLE saisie.v_synthese_vertebre TO grp_consult;
|
||||
GRANT ALL ON TABLE saisie.v_synthese_vertebre TO grp_admin;
|
||||
"""
|
||||
with con_sicen.begin() as cnx:
|
||||
cnx.execute(drop_v_vert)
|
||||
cnx.execute(v_synthese_vertebre)
|
||||
cnx.execute(grant_v_vert)
|
||||
|
||||
|
||||
drop_v_flo = "DROP VIEW IF EXISTS saisie.v_synthese_flore"
|
||||
v_synthese_flore = """
|
||||
CREATE OR REPLACE VIEW saisie.v_synthese_flore
|
||||
AS WITH update_data AS (
|
||||
SELECT
|
||||
vm.id_obs,
|
||||
MAX(s.date_operation) date_update
|
||||
FROM saisie.vm_synthese_observations vm
|
||||
LEFT JOIN saisie.suivi_saisie_observation s USING (id_obs)
|
||||
GROUP BY vm.id_obs
|
||||
)
|
||||
SELECT
|
||||
vm.id_obs,
|
||||
vm.etude,
|
||||
vm.protocole,
|
||||
vm.lot_donnee,
|
||||
s.date_operation,
|
||||
ud.date_update,
|
||||
vm.date_obs::date,
|
||||
vm.date_debut_obs::date,
|
||||
vm.date_fin_obs::date,
|
||||
vm.heure_obs::time,
|
||||
vm.regne,
|
||||
vm.classe,
|
||||
vm.ordre,
|
||||
vm.famille,
|
||||
vm.group1_inpn,
|
||||
vm.group2_inpn,
|
||||
vm.nom_latin,
|
||||
vm.nom_vern,
|
||||
vm.nom_complet,
|
||||
vm.cd_nom,
|
||||
vm.cd_ref,
|
||||
vm.absence_observation,
|
||||
vm.effectif,
|
||||
vm.effectif_min,
|
||||
vm.effectif_max,
|
||||
vm.effectif_textuel,
|
||||
vm.strate_flore,
|
||||
vm.phenologie_flore,
|
||||
vm.age_faune,
|
||||
vm.sexe_faune,
|
||||
vm.reprostatut_faune,
|
||||
vm.determination,
|
||||
vm.rmq_observation,
|
||||
vm.observateurs,
|
||||
vm.observateurs_v2,
|
||||
vm.numerisateur,
|
||||
vm.structures,
|
||||
vm.diffusable,
|
||||
vm.statut_validation,
|
||||
vm.validateur,
|
||||
vm.decision_validation,
|
||||
vm.insee_commune,
|
||||
vm.nom_commune,
|
||||
vm.lieu_dit,
|
||||
vm.altitude_z,
|
||||
vm.longitude_x,
|
||||
vm.latitude_y,
|
||||
vm."precision",
|
||||
vm.rmq_localisation,
|
||||
vm.geom,
|
||||
so.id_origine
|
||||
FROM saisie.vm_synthese_observations vm
|
||||
LEFT JOIN ref_inpn_taxref.taxref_v11 tax ON vm.cd_nom::integer = tax.cd_nom
|
||||
JOIN saisie.saisie_observation so USING (id_obs)
|
||||
LEFT JOIN update_data ud USING (id_obs)
|
||||
LEFT JOIN (SELECT * FROM saisie.suivi_saisie_observation WHERE operation = 'INSERT'::text) s USING (id_obs)
|
||||
WHERE tax.regne <> 'Animalia'::text OR (vm.cd_nom::text = ANY (ARRAY['9999008'::text, '9999003'::text, '9999005'::text, '9999006'::text, '9999004'::text, '9999009'::text, '9999010'::text, '9999011'::text, '9999013'::text, '9999014'::text, '9999015'::text, '9999016'::text, '9999017'::text, '9999018'::text, '9999019'::text, '9999020'::text, '9999021'::text, '9999007'::text, '9999012'::text, '9999001'::text, '9999002'::text]));
|
||||
"""
|
||||
grant_v_flo = """
|
||||
ALTER TABLE saisie.v_synthese_flore OWNER TO cen_admin;
|
||||
GRANT ALL ON TABLE saisie.v_synthese_flore TO cen_admin;
|
||||
GRANT SELECT ON TABLE saisie.v_synthese_flore TO grp_consult;
|
||||
GRANT ALL ON TABLE saisie.v_synthese_flore TO grp_admin;
|
||||
"""
|
||||
with con_sicen.begin() as cnx:
|
||||
cnx.execute(drop_v_flo)
|
||||
cnx.execute(v_synthese_flore)
|
||||
cnx.execute(grant_v_flo)
|
||||
Reference in New Issue
Block a user