update

0_FONCIER/foncier_call_fonctions.py

@@ -21,8 +21,8 @@ from pycen import con_fon
 epsg = '2154'
 crs = 'EPSG:%s'%epsg
 chunk = None
-pci_annee = '2020'
+pci_annee = '2021'
-matrice_annee = '2020'
+matrice_annee = '2021'
 start_time = dt.datetime.today()
 def time_exec (init_time):
   time = dt.datetime.today() - init_time
@@ -46,6 +46,7 @@ def get_data(table,schema,engine=con_fon,chunk=chunk):
     
     return df
 
+if __name__ == "__main__":
 
     ###############
     # Get sites
@@ -57,7 +58,7 @@ print('''IMPORT data from schema: '%s' , table : '%s' '''%(sch_site,tab_site))
     # sql = "SELECT * FROM {0}.{1}".format(sch_site, tab_site)
     sql = '''
         SELECT * FROM {0}.{1} 
-    WHERE site_id = 'GDMA'
+        WHERE site_id = 'CRAS'
         --WHERE site_id NOT IN (SELECT DISTINCT site_id FROM {2}.{3})
         '''.format(sch_site, tab_site, sch_fon, tab_fon)
     site = gpd.read_postgis(

0_FONCIER/foncier_insert_administratif.py

@@ -43,7 +43,7 @@ base   = 'bd_cen'
 #   pwd    = pwd,
 #   adr    = adr,
 #   base   = base
-#   # schema = schema
+#   schema = schema
 #   )
 crs = 'EPSG:2154'
 

0_FONCIER/foncier_insert_cadastre_V3.py

@@ -55,9 +55,9 @@ def _where_parcelle(sql0,schema,list_parid):
             list_parid = [list_parid]
         
         LIST_ID = str(tuple(list_parid)).replace(',)',')')
-
+        sql1 = 'AND' if 'WHERE' in sql0 else 'WHERE'
-        sql1 = '''
+        sql1 += '''
-        WHERE p.parcelle IN {list_id}
+         p.parcelle IN {list_id}
         ;'''.format(sch=schema,list_id=LIST_ID)
     else : 
         chunk = 200000
@@ -325,7 +325,7 @@ def _get_lots1(schema='38_202207'):
 
 
 def _get_lots2(schema='38_202207',list_parid=None):
-    sql0 = '''
+    sql0 = '''set work_mem='265MB';
     SELECT DISTINCT
         CASE WHEN TRIM(t.dnulot) = '' OR TRIM(t.dnulot) IS NULL
             THEN substring(t.parcelle from 1 for 2)||substring(t.parcelle from 4 for 12)||TRIM(t.ccosub)

3_AZALEE/create_view_zh3.py

@@ -511,7 +511,7 @@ comment = """
 COMMENT ON COLUMN zones_humides.v_zoneshumides.site_code IS 'Identifiant de la zone humide.';
 COMMENT ON COLUMN zones_humides.v_zoneshumides.old_code IS 'Ancien identifiant de la zone humide pouvant se retrouver dans des données historiques.';
 COMMENT ON COLUMN zones_humides.v_zoneshumides.nom IS 'Nom de la zone humide.';
-COMMENT ON COLUMN zones_humides.v_zoneshumides.autre_nom IS 'Autre nom de la zone humide.';
+COMMENT ON COLUMN zones_humides.v_zoneshumides.autre_nom IS 'Autre nom possible de la zone humide.';
 COMMENT ON COLUMN zones_humides.v_zoneshumides.auteur_site IS 'Auteur ayant caractérisé la zone humide pour la première fois.';
 COMMENT ON COLUMN zones_humides.v_zoneshumides.auteur_geom IS 'Auteur ayant définis la géometrie actuelle de la zone humide.';
 COMMENT ON COLUMN zones_humides.v_zoneshumides.auteur_last_maj IS 'Auteur le plus récent ayant défini les attributs de la zone humide.';
@@ -521,6 +521,7 @@ COMMENT ON COLUMN zones_humides.v_zoneshumides.date_last_maj IS 'Date des attrib
 COMMENT ON COLUMN zones_humides.v_zoneshumides.type_milieu IS 'Caractérisation du milieu.';
 COMMENT ON COLUMN zones_humides.v_zoneshumides.type_site IS '';
 COMMENT ON COLUMN zones_humides.v_zoneshumides.typo_sdage IS 'Typologie sdage de la zone humide.';
+COMMENT ON COLUMN zones_humides.v_zoneshumides.mnemo_sdage IS 'Libellé mnémonique sdage de la zone humide.';
 COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_site IS 'Remarques générale concernant la zone humide.';
 COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_fct_majeur IS 'Remarques sur les fonctions majeurs de la zone humide.';
 COMMENT ON COLUMN zones_humides.v_zoneshumides.rmq_interet_patri IS 'Remarques sur les intérêts patrimoniaux de la zone humide.';
@@ -549,6 +550,8 @@ COMMENT ON COLUMN zones_humides.v_zoneshumides.entree_eau_topo IS 'Liste de la t
 COMMENT ON COLUMN zones_humides.v_zoneshumides.sortie_eau_reg IS 'Liste des sorties d''eau du régime hydrique de la zone humide. Chaque élément de la liste est séparé par un  '';''.';
 COMMENT ON COLUMN zones_humides.v_zoneshumides.sortie_eau_perm IS 'Liste des permanances respectivement de chaque sortie d''eau de la zone humide. Chaque élément de la liste est séparé par un  '';''.';
 COMMENT ON COLUMN zones_humides.v_zoneshumides.sortie_eau_topo IS 'Liste de la toponymie respectivement de chaque sortie d''eau de la zone humide. Chaque élément de la liste est séparé par un  '';''.';
+COMMENT ON COLUMN zones_humides.v_zoneshumides.id_origine IS 'Identifiant d''origine de la zone fournie par un partenaire exterieur.';
+COMMENT ON COLUMN zones_humides.v_zoneshumides.geom IS 'Géometrie de la zone humide.';
 """
 with con.begin() as cnx:
     cnx.execute(text(v_zoneshumides))

3_AZALEE/tmp/PNE_extract_zh.py

@@ -28,5 +28,5 @@ if __name__ == "__main__":
     format_data(vzh_pne)
     format_data(zhp_pne)
 
-    vzh_pne.drop('old_code',axis=1).to_file(os.path.join(PATH,'PNE_ZH.gpkg'),driver='GPKG',layer='inventaire_zh')
+    vzh_pne.drop(['old_code','type_site'],axis=1).to_file(os.path.join(PATH,'PNE_ZH.gpkg'),driver='GPKG',layer='inventaire_zh')
     zhp_pne.to_file(os.path.join(PATH,'PNE_ZH.gpkg'),driver='GPKG',layer='zh_ponctuelle')

5_GEONATURE/ENS_extract_obs.py

@@ -2,9 +2,9 @@ from pycen import con_gn
 from pycen.wfs import list_layer,get_wfs
 import geopandas as gpd
 
-def extract_synthese_gn(table='v_synthese_for_export',con=con_gn,date_min=None,date_max=None,polygon=None):
+def extract_synthese_gn(table='v_synthese_for_export',con=con_gn,date_min=None,date_max=None,polygon=None,filter_obscen=True):
     sql = 'SELECT *,ST_GeomFromText(geometrie_wkt_4326,4326) geom FROM gn_synthese.{tab}'.format(tab=table)
-    if any([i is not None for i in [date_min,date_max,polygon]]):
+    if any([i is not None for i in [date_min,date_max,polygon]]) or filter_obscen:
         sql += ' WHERE '
         if date_min is not None:
             sql += "date_debut > '%s' AND " % date_min
@@ -12,7 +12,7 @@ def extract_synthese_gn(table='v_synthese_for_export',con=con_gn,date_min=None,d
             sql += "date_debut < '%s' AND " % date_max
         if polygon is not None:
             sql += "ST_Intersects(ST_Transform(ST_GeomFromText(geometrie_wkt_4326,4326),2154),'SRID={epsg};{poly}') AND ".format(epsg=2154,poly=polygon)
-
+        if filter_obscen:
             sql += """(
                 unaccent(observateurs) LIKE ANY (array['%%Conservatoire d%%Espaces Naturels Isere%%', '%%Conservatoire d%%Espaces Naturels d%%Isere%%', '%%CEN Isere%%', '%%Association des Amis de l%%Ile de la Platiere%%']) 
                 OR observateurs NOT LIKE '%%(%%)%%'

5_GEONATURE/pivot_bdc_status_v2.py

@@ -1,10 +1,9 @@
 from pycen import con_gn
 import numpy as np
 import pandas as pd
+import os
 
-# Liste des CD_NOM en entrée
+def get_status(lst):
-taxlist = pd.read_excel('/media/colas/SRV/FICHIERS/TRANSFERTS-EQUIPE/CG/BOCA_CD_nom.xlsx')
-
     sql = """
         SELECT 
             t.cd_nom,
@@ -27,9 +26,22 @@ sql = """
         FROM taxonomie.v_taxref_all_listes t
         JOIN taxonomie.v_bdc_status s USING (cd_nom)
         WHERE t.cd_nom IN {cd_nom}
-;""".format(cd_nom = tuple(taxlist.CD_NOM.astype(str)) )
+    ;""".format(cd_nom = tuple(lst))
-df = pd.read_sql_query(sql,con_gn)
+    return pd.read_sql_query(sql,con_gn)
-df.to_csv('/media/colas/SRV/FICHIERS/TRANSFERTS-EQUIPE/LC/BOCA_CD_NOM_STATUS.csv')
+    
+
+if __name__ == "__main__":
+
+    PATH = '/media/colas/Disk2/tmp/NICO'
+    file = 'MAIR_FAUNE&FLORE_PG2024_V0.xlsx'
+    sheet = 'MAIR_FLORE'
+
+    # Liste des CD_NOM en entrée
+    taxlist = pd.read_excel(os.path.join(PATH,file),sheet,usecols=['cd_nom'],header=1)
+    df = get_status(taxlist.cd_nom.astype(str))
+
+
+    # df.to_csv('/media/colas/SRV/FICHIERS/TRANSFERTS-EQUIPE/LC/BOCA_CD_NOM_STATUS.csv')
 
     pivot = pd.pivot_table(
         df,
@@ -41,6 +53,7 @@ pivot = pd.pivot_table(
 
     for c in pivot.columns:
         pivot[c] = [x[0] if x is not np.NaN and len(x)==1 else x for x in pivot[c]]
+    if 'DH' in pivot.columns:
         pivot['DH'] = [','.join(x) if (x is not np.NaN) and (len(x)==2) else x for x in pivot['DH']]
         pivot.DH.replace({'CDH':''},regex=True,inplace=True)
 
@@ -54,12 +67,13 @@ pivlib = pd.pivot_table(
         aggfunc=list,fill_value=None)
     for c in pivlib.columns:
         pivlib[c] = [x[0] if x is not np.NaN and len(x)==1 else x for x in pivlib[c]]
+    if 'DH' in pivot.columns:
         pivlib['DH'] = [','.join(x) if (x is not np.NaN) and (len(x)==2) else x for x in pivlib['DH']]
         pivlib.DH.replace({'CDH':''},regex=True,inplace=True)
 
 
     print('INIT writer')
-NAME_OUT = '/media/colas/SRV/FICHIERS/TRANSFERTS-EQUIPE/LC/BOCA_CD_NOM_STATUS.xlsx'
+    NAME_OUT = os.path.join(PATH,sheet+'_status.xlsx')
     with pd.ExcelWriter(NAME_OUT) as writer:
         df.to_excel(
             writer,sheet_name='v_bdc_status'
@@ -74,5 +88,6 @@ with pd.ExcelWriter(NAME_OUT) as writer:
         pivlib.to_excel(
             writer,sheet_name='pivot_libel'
             )
-    writer.save()
+        # writer.save()
         print('pivot_libel OK !')
+

6_SQL/manage_user.py

@@ -4,7 +4,6 @@
 from sqlalchemy.engine import URL
 from sqlalchemy import create_engine,text
 import pandas as pd
-from pycen import con_bdcen,con_fon
 
 # Parametres bdd
 # user = 'cen_admin'
@@ -68,8 +67,25 @@ def create_grp(con,grp):
     with con.begin() as cnx:
         cnx.execute(sql)
 
+def create_usr(con,usr,pwd):
+    sql = """CREATE USER "{usr}" WITH 
+        NOSUPERUSER
+        NOCREATEDB
+        NOCREATEROLE
+        INHERIT
+        LOGIN
+        NOREPLICATION
+        NOBYPASSRLS
+        PASSWORD '{pwd}'
+        CONNECTION LIMIT -1;""".format(usr=usr,pwd=pwd)
+    with con.begin() as cnx:
+        cnx.execute(sql)
 # sql = "select * from pg_catalog.pg_user" # where tableowner = 'gpasquier'"
 
+
+if __name__ == "__main__":
+    from pycen import con_bdcen,con_fon,con
+
     for grp in usr.keys():
         # create grp_role
         create_grp(con_fon,grp)

create_mbtile.py

@@ -38,13 +38,15 @@ def gen_mbtiles2(xml,output,format='mbtiles'):
     # projwin Isère : 836250 6534274 965160 6403454
     init = dt.now()
     cmd = '''
-        gdal_translate -outsize 75000 75000 -projwin 836250 6534274 965160 6403454 -projwin_srs "EPSG:2154" -of %s -co "COMPRESS=YES" -co "TILE_FORMAT=JPEG" -co "QUALITY=100" -co "MINZOOM=12" -co "MAXZOOM=22" --config GDAL_CACHEMAX 64 "%s" "%s"
+        gdal_translate -outsize 75000 75000 -projwin 836250 6534274 965160 6403454 -projwin_srs "EPSG:2154" -of %s -r cubic -co "COMPRESS=JPEG" -co "TILE_FORMAT=JPEG" -co "QUALITY=80" -co "MINZOOM=12" -co "MAXZOOM=22" -co "PHOTOMETRIC=YCBCR" --config GDAL_CACHEMAX 512 "%s" "%s"
         ''' % (format,xml,output)
     print(cmd)
     os.system(cmd)
     print(dt.now()-init)
 
 def gdalwarp(vrt,mbtiles):
+    # exemple : Découpage d'un raster à partir d'un polygon
+    # gdalwarp -overwrite -s_srs "EPSG:2154" -t_srs "EPSG:2154" -of GTiff -cutline "PG:dbname='azalee' host=91.134.194.221 port=5432 sslmode=disable user='cgeier' password='adm1n*bdCen'" -cl "ref_territoire.isere_platiere" -crop_to_cutline -multi -co "COMPRESS=JPEG" -co "JPEG_QUALITY=75" "/media/colas/Disk2/5_BDD/ZH_prob/ZH_probS_Auvergne-Rhone-Alpes/ZH_probS_region84.tif" "/media/colas/Disk2/5_BDD/ZH_prob/ZH_probS_Auvergne-Rhone-Alpes/ZH_probS_dept38.tif"
     cmd = '''
     gdalwarp -of MBTiles -s_srs epsg:2154 -t_srs epsg:3857 %s %s
     ''' % (vrt,mbtiles)
@@ -85,10 +87,10 @@ if __name__ == "__main__":
         output=os.path.join(PATH,xml),
         type_flux='wms')
 
-    gen_mbtiles2(os.path.join(PATH,xml),os.path.join(PATH,mbtile),format='mbtiles')
+    # gen_mbtiles2(os.path.join(PATH,xml),os.path.join(PATH,mbtile),format='mbtiles')
     # gen_mbtiles2(os.path.join(PATH,xml),os.path.join(PATH,vrt),format='vrt')
     # gdalwarp(os.path.join(PATH,vrt),os.path.join(PATH,mbtile))
-    # gdaladdo(os.path.join(PATH,mbtile))
+    gdaladdo(os.path.join(PATH,mbtile))
 
     # ds = gdal.Open(os.path.join(PATH,vrt))
 

raster2pgsql_multithread.py

@@ -5,19 +5,27 @@ from subprocess import run
 from os import listdir,getcwd,chdir,system
 
 
-def thread_function(name):
+def thread_func(name,table,schema):
     logging.info("Thread %s: starting", name)
     time.sleep(2)
     if not isinstance(name,list) : name = [name]
-    cmd = 'export PGPASSWORD=#CEN38@venir;raster2pgsql -s 2154 -a -t 5x5  {} ref_territoire.mnt_5m | psql -h 91.134.194.221 -U cen_admin -d azalee'.format(" ".join(name))
+    cmd = 'export PGPASSWORD=#CEN38@venir;raster2pgsql -s 2154 -c -C -I -M -t 5x5 -N 0  {file} {sch}.{tab} | psql -h 91.134.194.221 -U cen_admin -d azalee'.format(file=" ".join(name),sch=schema,tab=table)
     system(cmd)
     logging.info("Thread %s: finishing", name)
 
-def last_thread_function(name):
+def thread_function(name,table,schema):
     logging.info("Thread %s: starting", name)
     time.sleep(2)
     if not isinstance(name,list) : name = [name]
-    cmd = 'export PGPASSWORD=#CEN38@venir;raster2pgsql -s 2154 -a -C -I -M -t 5x5  {} ref_territoire.mnt_5m | psql -h 91.134.194.221 -U cen_admin -d azalee'.format(" ".join(name))
+    cmd = 'export PGPASSWORD=#CEN38@venir;raster2pgsql -s 2154 -a -t 5x5  {file} {sch}.{tab} | psql -h 91.134.194.221 -U cen_admin -d azalee'.format(file=" ".join(name),sch=schema,tab=table)
+    system(cmd)
+    logging.info("Thread %s: finishing", name)
+
+def last_thread_function(name,table,schema):
+    logging.info("Thread %s: starting", name)
+    time.sleep(2)
+    if not isinstance(name,list) : name = [name]
+    cmd = 'export PGPASSWORD=#CEN38@venir;raster2pgsql -s 2154 -a -C -I -M -t 5x5  {file} {sch}.{tab} | psql -h 91.134.194.221 -U cen_admin -d azalee'.format(file=" ".join(name),sch=schema,tab=table)
     system(cmd)
     logging.info("Thread %s: finishing", name)
 
@@ -45,9 +53,9 @@ if __name__ == "__main__":
         for file in list_asc[j:k]:
             logging.info("Main    : create and start thread %s.", file)
             if file == list_asc[-1]:
-                x = threading.Thread(target=last_thread_function, args=(file,))
+                x = threading.Thread(target=last_thread_function, args=(file,'mnt_5m','ref_territoire'))
             else:
-                x = threading.Thread(target=thread_function, args=(file,))
+                x = threading.Thread(target=thread_function, args=(file,'mnt_5m','ref_territoire'))
             threads.append(x)
             x.start()
         # [t.start() for t in threads]