Source code for aiida.backends.djsite.db.migrations.0024_dblog_update

# -*- coding: utf-8 -*-
###########################################################################
# Copyright (c), The AiiDA team. All rights reserved.                     #
# This file is part of the AiiDA code.                                    #
#                                                                         #
# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core #
# For further information on the license, see the LICENSE.txt file        #
# For further information please visit http://www.aiida.net               #
###########################################################################
# Generated by Django 1.11.16 on 2018-12-21 10:56
# pylint: disable=invalid-name
"""Migration for the update of the DbLog table. Addition of uuids"""

import sys
import click

# Remove when https://github.com/PyCQA/pylint/issues/1931 is fixed
# pylint: disable=no-name-in-module,import-error
from django.db import migrations, models
from aiida.backends.djsite.db.migrations import upgrade_schema_version
from aiida.backends.general.migrations.utils import dumps_json
from aiida.common.utils import get_new_uuid
from aiida.manage import configuration

REVISION = '1.0.24'
DOWN_REVISION = '1.0.23'

# The values that will be exported for the log records that will be deleted
values_to_export = ['id', 'time', 'loggername', 'levelname', 'objpk', 'objname', 'message', 'metadata']

node_prefix = 'node.'
leg_workflow_prefix = 'aiida.workflows.user.'


[docs]def get_legacy_workflow_log_number(schema_editor): """ Get the number of the log records that correspond to legacy workflows """ with schema_editor.connection.cursor() as cursor: cursor.execute( """ SELECT COUNT(*) FROM db_dblog WHERE (db_dblog.objname LIKE 'aiida.workflows.user.%') """ ) return cursor.fetchall()[0][0]
[docs]def get_unknown_entity_log_number(schema_editor): """ Get the number of the log records that correspond to unknown entities """ with schema_editor.connection.cursor() as cursor: cursor.execute( """ SELECT COUNT(*) FROM db_dblog WHERE (db_dblog.objname NOT LIKE 'node.%') AND (db_dblog.objname NOT LIKE 'aiida.workflows.user.%') """ ) return cursor.fetchall()[0][0]
[docs]def get_logs_with_no_nodes_number(schema_editor): """ Get the number of the log records that don't correspond to a node """ with schema_editor.connection.cursor() as cursor: cursor.execute( """ SELECT COUNT(*) FROM db_dblog WHERE (db_dblog.objname LIKE 'node.%') AND NOT EXISTS (SELECT 1 FROM db_dbnode WHERE db_dbnode.id = db_dblog.objpk LIMIT 1) """ ) return cursor.fetchall()[0][0]
[docs]def get_serialized_legacy_workflow_logs(schema_editor): """ Get the serialized log records that correspond to legacy workflows """ with schema_editor.connection.cursor() as cursor: cursor.execute(( """ SELECT db_dblog.id, db_dblog.time, db_dblog.loggername, db_dblog.levelname, db_dblog.objpk, db_dblog.objname, db_dblog.message, db_dblog.metadata FROM db_dblog WHERE (db_dblog.objname LIKE 'aiida.workflows.user.%') """ )) keys = ['id', 'time', 'loggername', 'levelname', 'objpk', 'objname', 'message', 'metadata'] res = list() for row in cursor.fetchall(): res.append(dict(list(zip(keys, row)))) return dumps_json(res)
[docs]def get_serialized_unknown_entity_logs(schema_editor): """ Get the serialized log records that correspond to unknown entities """ with schema_editor.connection.cursor() as cursor: cursor.execute(( """ SELECT db_dblog.id, db_dblog.time, db_dblog.loggername, db_dblog.levelname, db_dblog.objpk, db_dblog.objname, db_dblog.message, db_dblog.metadata FROM db_dblog WHERE (db_dblog.objname NOT LIKE 'node.%') AND (db_dblog.objname NOT LIKE 'aiida.workflows.user.%') """ )) keys = ['id', 'time', 'loggername', 'levelname', 'objpk', 'objname', 'message', 'metadata'] res = list() for row in cursor.fetchall(): res.append(dict(list(zip(keys, row)))) return dumps_json(res)
[docs]def get_serialized_logs_with_no_nodes(schema_editor): """ Get the serialized log records that don't correspond to a node """ with schema_editor.connection.cursor() as cursor: cursor.execute(( """ SELECT db_dblog.id, db_dblog.time, db_dblog.loggername, db_dblog.levelname, db_dblog.objpk, db_dblog.objname, db_dblog.message, db_dblog.metadata FROM db_dblog WHERE (db_dblog.objname LIKE 'node.%') AND NOT EXISTS (SELECT 1 FROM db_dbnode WHERE db_dbnode.id = db_dblog.objpk LIMIT 1) """ )) keys = ['id', 'time', 'loggername', 'levelname', 'objpk', 'objname', 'message', 'metadata'] res = list() for row in cursor.fetchall(): res.append(dict(list(zip(keys, row)))) return dumps_json(res)
[docs]def set_new_uuid(apps, _): """ Set new UUIDs for all logs """ DbLog = apps.get_model('db', 'DbLog') query_set = DbLog.objects.all() for log in query_set.iterator(): log.uuid = get_new_uuid() log.save(update_fields=['uuid'])
[docs]def export_and_clean_workflow_logs(apps, schema_editor): """ Export the logs records that correspond to legacy workflows and to unknown entities. """ from tempfile import NamedTemporaryFile DbLog = apps.get_model('db', 'DbLog') lwf_number = get_legacy_workflow_log_number(schema_editor) other_number = get_unknown_entity_log_number(schema_editor) log_no_node_number = get_logs_with_no_nodes_number(schema_editor) # If there are no legacy workflow log records or log records of an unknown entity if lwf_number == 0 and other_number == 0 and log_no_node_number == 0: return if not configuration.PROFILE.is_test_profile: click.echo( 'We found {} log records that correspond to legacy workflows and {} log records to correspond ' 'to an unknown entity.'.format(lwf_number, other_number) ) click.echo( 'These records will be removed from the database and exported to JSON files to the current directory).' ) proceed = click.confirm('Would you like to proceed?', default=True) if not proceed: sys.exit(1) delete_on_close = configuration.PROFILE.is_test_profile # Exporting the legacy workflow log records if lwf_number != 0: # Get the records and write them to file with NamedTemporaryFile( prefix='legagy_wf_logs-', suffix='.log', dir='.', delete=delete_on_close, mode='w+' ) as handle: filename = handle.name handle.write(get_serialized_legacy_workflow_logs(schema_editor)) # If delete_on_close is False, we are running for the user and add additional message of file location if not delete_on_close: click.echo(f'Exported legacy workflow logs to {filename}') # Now delete the records DbLog.objects.filter(objname__startswith=leg_workflow_prefix).delete() with schema_editor.connection.cursor() as cursor: cursor.execute(( """ DELETE FROM db_dblog WHERE (db_dblog.objname LIKE 'aiida.workflows.user.%') """ )) # Exporting unknown log records if other_number != 0: # Get the records and write them to file with NamedTemporaryFile( prefix='unknown_entity_logs-', suffix='.log', dir='.', delete=delete_on_close, mode='w+' ) as handle: filename = handle.name handle.write(get_serialized_unknown_entity_logs(schema_editor)) # If delete_on_close is False, we are running for the user and add additional message of file location if not delete_on_close: click.echo(f'Exported unexpected entity logs to {filename}') # Now delete the records DbLog.objects.exclude(objname__startswith=node_prefix).exclude(objname__startswith=leg_workflow_prefix).delete() with schema_editor.connection.cursor() as cursor: cursor.execute(( """ DELETE FROM db_dblog WHERE (db_dblog.objname NOT LIKE 'node.%') AND (db_dblog.objname NOT LIKE 'aiida.workflows.user.%') """ )) # Exporting log records that don't correspond to nodes if log_no_node_number != 0: # Get the records and write them to file with NamedTemporaryFile( prefix='no_node_entity_logs-', suffix='.log', dir='.', delete=delete_on_close, mode='w+' ) as handle: filename = handle.name handle.write(get_serialized_logs_with_no_nodes(schema_editor)) # If delete_on_close is False, we are running for the user and add additional message of file location if not delete_on_close: click.echo('Exported entity logs that don\'t correspond to nodes to {}'.format(filename)) # Now delete the records with schema_editor.connection.cursor() as cursor: cursor.execute(( """ DELETE FROM db_dblog WHERE (db_dblog.objname LIKE 'node.%') AND NOT EXISTS (SELECT 1 FROM db_dbnode WHERE db_dbnode.id = db_dblog.objpk LIMIT 1) """ ))
[docs]def clean_dblog_metadata(apps, _): """ Remove objpk and objname from the DbLog table metadata. """ import json DbLog = apps.get_model('db', 'DbLog') query_set = DbLog.objects.all() for log in query_set.iterator(): met = json.loads(log.metadata) if 'objpk' in met: del met['objpk'] if 'objname' in met: del met['objname'] log.metadata = json.dumps(met) log.save(update_fields=['metadata'])
[docs]def enrich_dblog_metadata(apps, _): """ Add objpk and objname to the DbLog table metadata. """ import json DbLog = apps.get_model('db', 'DbLog') query_set = DbLog.objects.all() for log in query_set.iterator(): met = json.loads(log.metadata) if 'objpk' not in met: met['objpk'] = log.objpk if 'objname' not in met: met['objname'] = log.objname log.metadata = json.dumps(met) log.save(update_fields=['metadata'])
[docs]class Migration(migrations.Migration): """ This migration updates the DbLog schema and adds UUID for correct export of the DbLog entries. More specifically, it adds UUIDS, it exports to files the not needed log entries (that correspond to legacy workflows and unknown entities), it creates a foreign key to the dbnode table, it transfers there the objpk data to the new dbnode column (just altering the objpk column and making it a foreign key when containing data, raised problems) and in the end objpk and objname columns are removed. """ dependencies = [ ('db', '0023_calc_job_option_attribute_keys'), ] operations = [ # Export of the logs of the old workflows to a JSON file, there is no re-import # for the reverse migrations migrations.RunPython(export_and_clean_workflow_logs, reverse_code=migrations.RunPython.noop), # Removing objname and objpk from the metadata. The reverse migration adds the # objname and objpk to the metadata migrations.RunPython(clean_dblog_metadata, reverse_code=enrich_dblog_metadata), # The forward migration will not do anything for the objname, the reverse # migration will populate it with correct values migrations.RunSQL( '', reverse_sql='UPDATE db_dblog SET objname=db_dbnode.type ' 'FROM db_dbnode WHERE db_dbnode.id = db_dblog.objpk' ), # Removal of the column objname, the reverse migration will add it migrations.RemoveField(model_name='dblog', name='objname'), # Creation of a new column called dbnode which is a foreign key to the dbnode table # The reverse migration will remove this column migrations.AddField( model_name='dblog', name='dbnode', field=models.ForeignKey( on_delete=models.deletion.CASCADE, related_name='dblogs', to='db.DbNode', blank=True, null=True ), ), # Transfer of the data from the objpk to the node field # The reverse migration will do the inverse transfer migrations.RunSQL('UPDATE db_dblog SET dbnode_id=objpk', reverse_sql='UPDATE db_dblog SET objpk=dbnode_id'), # Now that all the data have been migrated, make the column not nullable and not blank. # A log record should always correspond to a node record migrations.AlterField( model_name='dblog', name='dbnode', field=models.ForeignKey(on_delete=models.deletion.CASCADE, related_name='dblogs', to='db.DbNode'), ), # Since the new column is created correctly, drop the old objpk column # The reverse migration will add the field migrations.RemoveField(model_name='dblog', name='objpk'), # This is the correct pattern to generate unique fields, see # https://docs.djangoproject.com/en/1.11/howto/writing-migrations/#migrations-that-add-unique-fields # The reverse migration will remove it migrations.AddField( model_name='dblog', name='uuid', field=models.UUIDField(default=get_new_uuid, null=True), ), # Add unique UUIDs to the UUID field. There is no need for a reverse migration for a field # tha will be deleted migrations.RunPython(set_new_uuid, reverse_code=migrations.RunPython.noop), # Changing the column to unique migrations.AlterField( model_name='dblog', name='uuid', field=models.UUIDField(default=get_new_uuid, unique=True), ), upgrade_schema_version(REVISION, DOWN_REVISION) ]