concatenate-enrich-dois.py

import z_personal_functions as my_functions
import requests, json, random, pandas as pd


print("\n\nRunning concatenate-enrich-dois.py")

# ______0______ load DOIs and remove duplicate

## specifier la liste des entrepôts à importer
files_to_load = [ "zenodo", "datacite", "rdg", "bso-via-hal", "nakala" ] 

dois_raw = my_functions.from_files_load_dois(files_to_load)
print("\n\tDOIs loaded\t\t\t", len(dois_raw))

dois = list(set(dois_raw)) ## remove duplicate
print("\tDOIs to treat\t\t", len(dois))


# ______1_____ load DOIs already treater & get md from DataCite for new one

## pour essayer avec un seul DOI
# temp_doi = dois[random.randint(0, len(dois))]
# #temp_doi = "10.57745/QYIAWX" - 10.25656/01:8509
# print(temp_doi)
# raw_metadatas = my_functions.get_md_from_datacite(temp_doi)

doi_error = [] # retrieve doi error
temp_rows = [] # put data in dict before df

df_old = pd.read_csv("../dois-uga.csv")
print(f"\n\tnb of DOIs already treated\t{len(df_old)}")

# req dataCite and paste data following instructions
for doi in dois : #[:300]

	## if doi already treated
	if doi in df_old["doi"].values : 
		#print(f"\talready treated\t\t{doi}")
		continue

	## ___n___ get md from datacite
	raw_md = my_functions.get_md_from_datacite(doi)
	## to debug print(f"\t{doi}")

	### if doi not in datacite
	if raw_md == "error" : 
		doi_error.append(doi)
		continue

	## ___n___ from manual instructions retrieve appropriate data
	selected_md = my_functions.parse_datacite_md(raw_md) ## placer les resultats dans un dictionnaire
	temp_rows.append(selected_md) ## ajouter ce dictionnaire à une liste
	print(f"\tadded\t\t{doi}")


if temp_rows :
	df_fresh = pd.DataFrame(temp_rows)
	df_out = pd.concat([df_old, df_fresh], ignore_index=True)
	df_out.to_csv("../dois-uga.csv", index = False)
	print(f"\n\nnb of doi exported \t{len(df_out)}")