import os import pandas as pd import shutil from datetime import datetime from canvas_data.api import CanvasDataAPI import gzip # DOCUMENTAÇÃO DA BIBLIOTECA EM: # https://canvas-data-sdk.readthedocs.io/en/latest/canvas_data.html#module-canvas_data.api

canvas = CanvasDataAPI(os.environ["CANVAS_API_KEY"], os.environ["CANVAS_API_SECRET"])

df_downloaded_files = pd.read_parquet('/datasets/s3unaspdata/canvas/df_downloaded_files.parquet')

sync_files = canvas.get_sync_file_urls() schemaVersion = sync_files['schemaVersion'] path_full_files = '/datasets/s3unaspdata/canvas/full_files/' + schemaVersion.replace('.', '_') path_full_files_requests = '/datasets/s3unaspdata/canvas/full_files/' + schemaVersion.replace('.', '_') + '/requests' path_gz_full_files = '/datasets/s3unaspdata/canvas/gz_full_files' # salva data de último download f = open('/datasets/s3unaspdata/canvas/latest-get-full-files.txt', 'w+') f.write(str(datetime.now())) f.close() # print('limpando pastas') shutil.rmtree('./error/' + schemaVersion.replace('.', '_'), ignore_errors=True) shutil.rmtree(path_gz_full_files, ignore_errors=True) # shutil.rmtree(path_full_files, ignore_errors=True) # salva df com lista completa de arquivos do dump df_sync_files = pd.DataFrame(sync_files) df_sync_files.to_parquet('/datasets/s3unaspdata/canvas/gz_full_files_list.parquet') df_downloaded_files = pd.read_parquet('/datasets/s3unaspdata/canvas/df_downloaded_files.parquet') # baixa arquivos for file_obj in sync_files['files']: if('requests' not in file_obj['filename']): continue print('Iniciando download ' + schemaVersion + ' - ' + file_obj['filename']) if(len(df_downloaded_files.index) > 0 and len(df_downloaded_files.filename[df_downloaded_files.filename == str(file_obj['filename'])].index) > 0): print('Arquivo já processado ' + schemaVersion + ' - ' + file_obj['filename']) continue canvas.get_file(file_obj, download_directory=path_gz_full_files) arquivo_gz = file_obj['filename'] filepath_gz = path_gz_full_files + '/' + arquivo_gz filepath_txt = filepath_gz.replace('.gz', '.txt') print('Descompactando gzip') try : with gzip.open(filepath_gz, 'r') as f_in, open(filepath_txt, 'wb') as f_out: shutil.copyfileobj(f_in, f_out) if not os.path.exists(path_full_files): os.makedirs(path_full_files) try: df = pd.read_csv(filepath_txt, sep='\t', header=None, engine='python') except pd.errors.ParserError as erro: df = pd.read_table(filepath_txt, sep='\t', header=None) df = df.astype(str) if not os.path.exists(path_full_files_requests + '/csv'): os.makedirs(path_full_files_requests + '/csv') if not os.path.exists(path_full_files_requests + '/txt'): os.makedirs(path_full_files_requests + '/txt') df.to_csv(path_full_files_requests + '/csv/' + arquivo_gz.split('.gz')[0] + '.csv') shutil.copy(filepath_txt, path_full_files_requests + '/txt/' + arquivo_gz.split('.gz')[0] + '.txt') print(str(len(df.index)) + ' registros copiados') os.remove(filepath_gz) os.remove(filepath_txt) df_downloaded_files = df_downloaded_files.append(pd.DataFrame([str(file_obj['filename'])], columns=['filename']), ignore_index=True) df_downloaded_files.to_parquet('/datasets/s3unaspdata/canvas/df_downloaded_files.parquet') del df except gzip.BadGzipFile as erro : filepath_erro = './error/' + schemaVersion.replace('.', '_') if not os.path.exists(filepath_erro): os.makedirs(filepath_erro) shutil.move(filepath_gz, filepath_erro + '/' + arquivo_gz) os.remove(filepath_txt) print('Erro ao descompactar gzip, arquivo movido para: ' + filepath_erro + '/' + arquivo_gz) print('Download finalizado')