Schema Info

Schema Name: sfdl_dev_manual.genesys_inbound_call_segments

created: 2019-08-21 16:46:06
disabled: False
format: raw
id: 1288
is_partitioned: True
is_schema_datatyped: False

name:

{
    "database": "sfdl_dev_manual",
    "table": "genesys_inbound_call_segments"
}

type: manual

Schema Versions: 4

Version 4693285

created: 2023-03-09 14:57:31

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_manual`",
    "DROP TABLE IF EXISTS `sfdl_dev_manual.genesys_inbound_call_segments`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_manual.genesys_inbound_call_segments`\n(\n`participant_type`STRING,`participant_name`STRING,`agent_id`STRING,`participant_id`STRING,`conversation_start`STRING,`disconnect_type`STRING,`wrapup_code`STRING,`_origin`STRING,`direction`STRING,`segment_type`STRING,`received_number`STRING,`sequence`STRING,`queue_id`STRING,`start_time`STRING,`ivr_id`STRING,`conversation_end`STRING,`end_time`STRING,`origin_number`STRING,`flow_id`STRING,`conversation_id`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments'\n",
    "MSCK REPAIR TABLE sfdl_dev_manual.genesys_inbound_call_segments"
]

hive_path: s3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments
id: 4693285
processed: 1

raw_data:

{
    "_origin": "2023-03-09T14:56:35Z|separated_dt/inbound_call_segments/dt=2023-03-09/37799909-9b9c-4596-ad6c-f5b5aa114beb.json.gz|airflow_genesys_dev",
    "agent_id": null,
    "conversation_end": "2023-03-09T06:17:44.355Z",
    "conversation_id": "146d6294-0d66-42ac-a0a8-de0ec9c1c8aa",
    "conversation_start": "2023-03-09T06:17:32.083Z",
    "direction": "inbound",
    "disconnect_type": null,
    "end_time": "2023-03-09T06:17:32.178Z",
    "flow_id": "c52a2e71-7c67-4a50-9dbe-2461f87f0201",
    "ivr_id": "0000",
    "origin_number": "tel:+14128728373",
    "participant_id": "c5e98f46-5e59-4b75-910b-a9420b7128cf",
    "participant_name": "Pittsburgh Suburban PA",
    "participant_type": "external",
    "queue_id": null,
    "received_number": "tel:+15125186539",
    "segment_type": "system",
    "sequence": 0,
    "start_time": "2023-03-09T06:17:32.083Z",
    "wrapup_code": null
}

schema_attributes:

{
    "_origin": "string",
    "agent_id": "string",
    "conversation_end": "string",
    "conversation_id": "string",
    "conversation_start": "string",
    "direction": "string",
    "disconnect_type": "string",
    "end_time": "string",
    "flow_id": "string",
    "ivr_id": "string",
    "origin_number": "string",
    "participant_id": "string",
    "participant_name": "string",
    "participant_type": "string",
    "queue_id": "string",
    "received_number": "string",
    "segment_type": "string",
    "sequence": "string",
    "start_time": "string",
    "wrapup_code": "string"
}

schema_name_id: 1288
schema_scan_id: 6134271
updated: 2023-03-09 14:58:08

Version 4693284

created: 2023-03-09 14:57:31

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_manual`",
    "DROP TABLE IF EXISTS `sfdl_dev_manual.genesys_inbound_call_segments`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_manual.genesys_inbound_call_segments`\n(\n`participant_type`STRING,`participant_name`STRING,`agent_id`STRING,`participant_id`STRING,`conversation_start`STRING,`disconnect_type`STRING,`wrapup_code`STRING,`_origin`STRING,`direction`STRING,`segment_type`STRING,`received_number`STRING,`sequence`STRING,`queue_id`STRING,`start_time`STRING,`ivr_id`STRING,`conversation_end`STRING,`end_time`STRING,`origin_number`STRING,`flow_id`STRING,`conversation_id`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments'\n",
    "MSCK REPAIR TABLE sfdl_dev_manual.genesys_inbound_call_segments"
]

hive_path: s3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments
id: 4693284
processed: 1

raw_data:

{
    "_origin": "2023-03-09T14:56:33Z|separated_dt/inbound_call_segments/dt=2023-03-08/19f86570-8949-485f-96ca-ce10a13197be.json.gz|airflow_genesys_dev",
    "agent_id": null,
    "conversation_end": "2023-03-08T12:24:03.194Z",
    "conversation_id": "a005fa10-bcd5-42b2-bc02-b4741eed93e3",
    "conversation_start": "2023-03-08T05:04:28.192Z",
    "direction": "inbound",
    "disconnect_type": null,
    "end_time": "2023-03-08T05:04:28.272Z",
    "flow_id": "c52a2e71-7c67-4a50-9dbe-2461f87f0201",
    "ivr_id": "0000",
    "origin_number": "tel:+19705497652",
    "participant_id": "7cd2e846-fa6c-497b-ab09-10e53314b906",
    "participant_name": "Grand Junction CO",
    "participant_type": "customer",
    "queue_id": null,
    "received_number": "tel:+15122775732",
    "segment_type": "system",
    "sequence": 0,
    "start_time": "2023-03-08T05:04:28.192Z",
    "wrapup_code": null
}

schema_attributes:

{
    "_origin": "string",
    "agent_id": "string",
    "conversation_end": "string",
    "conversation_id": "string",
    "conversation_start": "string",
    "direction": "string",
    "disconnect_type": "string",
    "end_time": "string",
    "flow_id": "string",
    "ivr_id": "string",
    "origin_number": "string",
    "participant_id": "string",
    "participant_name": "string",
    "participant_type": "string",
    "queue_id": "string",
    "received_number": "string",
    "segment_type": "string",
    "sequence": "string",
    "start_time": "string",
    "wrapup_code": "string"
}

schema_name_id: 1288
schema_scan_id: 6134270
updated: 2023-03-09 14:58:10

Version 4672173

created: 2019-09-10 12:30:44

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_manual`",
    "DROP TABLE IF EXISTS `sfdl_dev_manual.genesys_inbound_call_segments`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_manual.genesys_inbound_call_segments`\n(\n`agent_id`STRING,`participant_name`STRING,`_origin`STRING,`conversation_start`STRING,`conversation_end`STRING,`participant_type`STRING,`conversation_id`STRING,`direction`STRING,`ivr_id`STRING,`queue_id`STRING,`origin_number`STRING,`received_number`STRING,`segment_type`STRING,`end_time`STRING,`sequence`STRING,`disconnect_type`STRING,`start_time`STRING,`flow_id`STRING,`participant_id`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments'\n",
    "MSCK REPAIR TABLE sfdl_dev_manual.genesys_inbound_call_segments"
]

hive_path: s3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments
id: 4672173
processed: 1

raw_data:

{
    "_origin": "2019-09-10T12:30:17Z|separated_dt/inbound_call_segments/dt=2019-09-10/2db08e0e-3027-40c9-a4db-7f7e76086bc1.json.gz|airflow_genesys_dev",
    "agent_id": null,
    "conversation_end": "2019-09-10T15:43:51.705Z",
    "conversation_id": "616b14d1-604c-4b49-a84a-b63eee5af667",
    "conversation_start": "2019-09-10T15:21:58.889Z",
    "direction": "inbound",
    "disconnect_type": null,
    "end_time": "2019-09-10T15:21:58.956Z",
    "flow_id": null,
    "ivr_id": "0000",
    "origin_number": "tel:+15037693436",
    "participant_id": "5c755431-5f18-4176-ab17-01cce46b978b",
    "participant_name": "Stayton OR",
    "participant_type": "customer",
    "queue_id": null,
    "received_number": "tel:+15128863822",
    "segment_type": "system",
    "sequence": 0,
    "start_time": "2019-09-10T15:21:58.893Z"
}

schema_attributes:

{
    "_origin": "string",
    "agent_id": "string",
    "conversation_end": "string",
    "conversation_id": "string",
    "conversation_start": "string",
    "direction": "string",
    "disconnect_type": "string",
    "end_time": "string",
    "flow_id": "string",
    "ivr_id": "string",
    "origin_number": "string",
    "participant_id": "string",
    "participant_name": "string",
    "participant_type": "string",
    "queue_id": "string",
    "received_number": "string",
    "segment_type": "string",
    "sequence": "string",
    "start_time": "string"
}

schema_name_id: 1288
schema_scan_id: 5165377
updated: 2019-09-10 12:30:58

Version 4671941

created: 2019-08-26 13:24:24

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_manual`",
    "DROP TABLE IF EXISTS `sfdl_dev_manual.genesys_inbound_call_segments`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_manual.genesys_inbound_call_segments`\n(\n`agent_id`STRING,`participant_name`STRING,`_origin`STRING,`conversation_start`STRING,`conversation_end`STRING,`participant_type`STRING,`conversation_id`STRING,`direction`STRING,`queue_id`STRING,`origin_number`STRING,`received_number`STRING,`segment_type`STRING,`end_time`STRING,`sequence`STRING,`disconnect_type`STRING,`start_time`STRING,`flow_id`STRING,`participant_id`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments'\n",
    "MSCK REPAIR TABLE sfdl_dev_manual.genesys_inbound_call_segments"
]

hive_path: s3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments
id: 4671941
processed: 1

raw_data:

{
    "_origin": "2019-08-26T13:23:57Z|separated_dt/inbound_call_segments/dt=2019-08-26/df74138a-ab43-411c-a8b8-941a4ef6b470.json.gz|airflow_genesys_dev",
    "agent_id": null,
    "conversation_end": "2019-08-26T14:00:46.344Z",
    "conversation_id": "288401c2-777f-426f-b389-20663923423e",
    "conversation_start": "2019-08-26T13:36:42.859Z",
    "direction": "inbound",
    "disconnect_type": null,
    "end_time": "2019-08-26T13:36:42.922Z",
    "flow_id": null,
    "origin_number": "tel:+13023166671",
    "participant_id": "18ebd39f-bd00-49ed-8f09-6c91e44ab022",
    "participant_name": "Millsboro DE",
    "participant_type": "customer",
    "queue_id": null,
    "received_number": "tel:+15128863822",
    "segment_type": "system",
    "sequence": 0,
    "start_time": "2019-08-26T13:36:42.863Z"
}

schema_attributes:

{
    "_origin": "string",
    "agent_id": "string",
    "conversation_end": "string",
    "conversation_id": "string",
    "conversation_start": "string",
    "direction": "string",
    "disconnect_type": "string",
    "end_time": "string",
    "flow_id": "string",
    "origin_number": "string",
    "participant_id": "string",
    "participant_name": "string",
    "participant_type": "string",
    "queue_id": "string",
    "received_number": "string",
    "segment_type": "string",
    "sequence": "string",
    "start_time": "string"
}

schema_name_id: 1288
schema_scan_id: 5148380
updated: 2019-08-26 13:24:37

Schema Scans: 4

Last at 2023-03-09 14:57:07

Scan 6134271

duration: 0:01:11.829527

exit_message:

{
    "ddl_changed": true,
    "ingested_partitions": 78,
    "partitions_applied": false,
    "success": true
}

id: 6134271

payload:

{
    "datatype_dict": null,
    "file_format": "json",
    "is_schema_datatyped": false,
    "partition": {
        "location": "s3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2023-03-09/",
        "name": "dt",
        "type": "string",
        "value": "2023-03-09"
    },
    "s3": {
        "bucket": "sfdl-manual-genesys-dev",
        "hive_path": "s3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments",
        "key": "separated_dt/inbound_call_segments/dt=2023-03-09/37799909-9b9c-4596-ad6c-f5b5aa114beb.json.gz"
    },
    "schema": {
        "name": "sfdl_dev_manual.genesys_inbound_call_segments",
        "type": "manual"
    }
}

running: False
schema_name_id: 1288
start_time: 2023-03-09 14:57:07
success: True
trace_id: 3165273381625361619

Scan 6134270

duration: 0:01:16.907869

exit_message:

{
    "exception": "failed to insert partition: (psycopg2.errors.UniqueViolation) duplicate key value violates unique constraint \"partitions_schema_name_id_value_name_key\"\nDETAIL:  Key (schema_name_id, value, name)=(1288, 2019-09-25, dt) already exists.\n\n[SQL: INSERT INTO partitions (schema_name_id, name, type, value, location, processed, created, data_first_arrived, data_last_arrived, parquet_first_performed, parquet_last_performed, parquet_in_process, parquet_job_id, flag_for_parquet) VALUES (%(schema_name_id)s, %(name)s, %(type)s, %(value)s, %(location)s, %(processed)s, %(created)s, %(data_first_arrived)s, %(data_last_arrived)s, %(parquet_first_performed)s, %(parquet_last_performed)s, %(parquet_in_process)s, %(parquet_job_id)s, %(flag_for_parquet)s) RETURNING partitions.id]\n[parameters: {'schema_name_id': 1288, 'name': 'dt', 'type': 'string', 'value': '2019-09-25', 'location': 's3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2019-09-25/', 'processed': 0, 'created': '2023-03-09 14:58:13.564453', 'data_first_arrived': '2023-03-09 14:58:13.564453', 'data_last_arrived': '2023-03-09 14:58:13.564453', 'parquet_first_performed': None, 'parquet_last_performed': None, 'parquet_in_process': None, 'parquet_job_id': None, 'flag_for_parquet': False}]\n(Background on this error at: https://sqlalche.me/e/14/gkpj)\nTraceback (most recent call last):\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py\", line 1900, in _execute_context\n    self.dialect.do_execute(\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/default.py\", line 736, in do_execute\n    cursor.execute(statement, parameters)\npsycopg2.errors.UniqueViolation: duplicate key value violates unique constraint \"partitions_schema_name_id_value_name_key\"\nDETAIL:  Key (schema_name_id, value, name)=(1288, 2019-09-25, dt) already exists.\n\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File \"/schema_manager/schema_manager/service/partition_bag.py\", line 114, in _insert\n    session.commit()\n  File \"\", line 2, in commit\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py\", line 1451, in commit\n    self._transaction.commit(_to_root=self.future)\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py\", line 829, in commit\n    self._prepare_impl()\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py\", line 808, in _prepare_impl\n    self.session.flush()\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py\", line 3429, in flush\n    self._flush(objects)\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py\", line 3568, in _flush\n    with util.safe_reraise():\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py\", line 70, in __exit__\n    compat.raise_(\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/util/compat.py\", line 208, in raise_\n    raise exception\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py\", line 3529, in _flush\n    flush_context.execute()\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/unitofwork.py\", line 456, in execute\n    rec.execute(self)\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/unitofwork.py\", line 630, in execute\n    util.preloaded.orm_persistence.save_obj(\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/persistence.py\", line 245, in save_obj\n    _emit_insert_statements(\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/persistence.py\", line 1238, in _emit_insert_statements\n    result = connection._execute_20(\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py\", line 1705, in _execute_20\n    return meth(self, args_10style, kwargs_10style, execution_options)\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/sql/elements.py\", line 333, in _execute_on_connection\n    return connection._execute_clauseelement(\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py\", line 1572, in _execute_clauseelement\n    ret = self._execute_context(\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py\", line 1943, in _execute_context\n    self._handle_dbapi_exception(\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py\", line 2124, in _handle_dbapi_exception\n    util.raise_(\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/util/compat.py\", line 208, in raise_\n    raise exception\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py\", line 1900, in _execute_context\n    self.dialect.do_execute(\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/default.py\", line 736, in do_execute\n    cursor.execute(statement, parameters)\nsqlalchemy.exc.IntegrityError: (psycopg2.errors.UniqueViolation) duplicate key value violates unique constraint \"partitions_schema_name_id_value_name_key\"\nDETAIL:  Key (schema_name_id, value, name)=(1288, 2019-09-25, dt) already exists.\n\n[SQL: INSERT INTO partitions (schema_name_id, name, type, value, location, processed, created, data_first_arrived, data_last_arrived, parquet_first_performed, parquet_last_performed, parquet_in_process, parquet_job_id, flag_for_parquet) VALUES (%(schema_name_id)s, %(name)s, %(type)s, %(value)s, %(location)s, %(processed)s, %(created)s, %(data_first_arrived)s, %(data_last_arrived)s, %(parquet_first_performed)s, %(parquet_last_performed)s, %(parquet_in_process)s, %(parquet_job_id)s, %(flag_for_parquet)s) RETURNING partitions.id]\n[parameters: {'schema_name_id': 1288, 'name': 'dt', 'type': 'string', 'value': '2019-09-25', 'location': 's3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2019-09-25/', 'processed': 0, 'created': '2023-03-09 14:58:13.564453', 'data_first_arrived': '2023-03-09 14:58:13.564453', 'data_last_arrived': '2023-03-09 14:58:13.564453', 'parquet_first_performed': None, 'parquet_last_performed': None, 'parquet_in_process': None, 'parquet_job_id': None, 'flag_for_parquet': False}]\n(Background on this error at: https://sqlalche.me/e/14/gkpj)\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File \"/schema_manager/schema_generator/schema_scanner.py\", line 102, in scan\n    response = service.run()\n  File \"/schema_manager/schema_generator/process_hive_table.py\", line 105, in run\n    response['ingested_partitions'] = pi.run()  # *total* current number of partitions in the table\n  File \"/schema_manager/schema_manager/service/partition_ingester.py\", line 29, in run\n    self.insert_and_clear_partition(schema_version.hive_path, row['partition'])\n  File \"/schema_manager/schema_manager/service/partition_ingester.py\", line 37, in insert_and_clear_partition\n    p = PartitionBagService.backfill_insert(self.schema_name, name, 'string', value, s3_location)\n  File \"/schema_manager/schema_manager/service/partition_bag.py\", line 60, in backfill_insert\n    return PartitionBagService._insert(schema_name, name, type, value, location)\n  File \"/schema_manager/schema_manager/service/partition_bag.py\", line 117, in _insert\n    raise PartitionBagException('failed to insert partition: {}'.format(e))\nschema_manager.service.partition_bag.PartitionBagException: failed to insert partition: (psycopg2.errors.UniqueViolation) duplicate key value violates unique constraint \"partitions_schema_name_id_value_name_key\"\nDETAIL:  Key (schema_name_id, value, name)=(1288, 2019-09-25, dt) already exists.\n\n[SQL: INSERT INTO partitions (schema_name_id, name, type, value, location, processed, created, data_first_arrived, data_last_arrived, parquet_first_performed, parquet_last_performed, parquet_in_process, parquet_job_id, flag_for_parquet) VALUES (%(schema_name_id)s, %(name)s, %(type)s, %(value)s, %(location)s, %(processed)s, %(created)s, %(data_first_arrived)s, %(data_last_arrived)s, %(parquet_first_performed)s, %(parquet_last_performed)s, %(parquet_in_process)s, %(parquet_job_id)s, %(flag_for_parquet)s) RETURNING partitions.id]\n[parameters: {'schema_name_id': 1288, 'name': 'dt', 'type': 'string', 'value': '2019-09-25', 'location': 's3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2019-09-25/', 'processed': 0, 'created': '2023-03-09 14:58:13.564453', 'data_first_arrived': '2023-03-09 14:58:13.564453', 'data_last_arrived': '2023-03-09 14:58:13.564453', 'parquet_first_performed': None, 'parquet_last_performed': None, 'parquet_in_process': None, 'parquet_job_id': None, 'flag_for_parquet': False}]\n(Background on this error at: https://sqlalche.me/e/14/gkpj)\n: Expecting value: line 1 column 1 (char 0)"
}

id: 6134270

payload:

{
    "datatype_dict": null,
    "file_format": "json",
    "is_schema_datatyped": false,
    "partition": {
        "location": "s3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2023-03-08/",
        "name": "dt",
        "type": "string",
        "value": "2023-03-08"
    },
    "s3": {
        "bucket": "sfdl-manual-genesys-dev",
        "hive_path": "s3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments",
        "key": "separated_dt/inbound_call_segments/dt=2023-03-08/19f86570-8949-485f-96ca-ce10a13197be.json.gz"
    },
    "schema": {
        "name": "sfdl_dev_manual.genesys_inbound_call_segments",
        "type": "manual"
    }
}

running: False
schema_name_id: 1288
start_time: 2023-03-09 14:56:56
success: False
trace_id: 7906921203109538404

Scan 5165377

duration: 0:00:16.886852

exit_message:

{
    "ddl_changed": true,
    "ingested_partitions": 8,
    "partitions_applied": false,
    "success": true
}

id: 5165377

payload:

{
    "datatype_dict": null,
    "file_format": "json",
    "is_schema_datatyped": false,
    "partition": {
        "location": "s3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2019-09-10/",
        "name": "dt",
        "type": "string",
        "value": "2019-09-10"
    },
    "s3": {
        "bucket": "sfdl-manual-genesys-dev",
        "hive_path": "s3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments",
        "key": "separated_dt/inbound_call_segments/dt=2019-09-10/2db08e0e-3027-40c9-a4db-7f7e76086bc1.json.gz"
    },
    "schema": {
        "name": "sfdl_dev_manual.genesys_inbound_call_segments",
        "type": "manual"
    }
}

running: False
schema_name_id: 1288
start_time: 2019-09-10 12:30:44
success: True
trace_id: 5032508889823830514

Scan 5148380

duration: 0:00:18.646722

exit_message:

{
    "ddl_changed": true,
    "ingested_partitions": 3,
    "partitions_applied": false,
    "success": true
}

id: 5148380

payload:

{
    "datatype_dict": null,
    "file_format": "json",
    "is_schema_datatyped": false,
    "partition": {
        "location": "s3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2019-08-26/",
        "name": "dt",
        "type": "string",
        "value": "2019-08-26"
    },
    "s3": {
        "bucket": "sfdl-manual-genesys-dev",
        "hive_path": "s3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments",
        "key": "separated_dt/inbound_call_segments/dt=2019-08-26/df74138a-ab43-411c-a8b8-941a4ef6b470.json.gz"
    },
    "schema": {
        "name": "sfdl_dev_manual.genesys_inbound_call_segments",
        "type": "manual"
    }
}

running: False
schema_name_id: 1288
start_time: 2019-08-26 13:24:23
success: True
trace_id: 1912558413981787765

Partitions:

count: 5

ddl:

[
    "ALTER TABLE\n    `sfdl_dev_manual.genesys_inbound_call_segments`\nADD\n\n\n PARTITION (dt='2024-05-11') location 's3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2024-05-11/'\n PARTITION (dt='2024-05-12') location 's3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2024-05-12/'\n PARTITION (dt='2024-05-13') location 's3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2024-05-13/'\n PARTITION (dt='2024-08-12') location 's3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2024-08-12/'\n PARTITION (dt='2024-08-13') location 's3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2024-08-13/'\n;"
]

name: dt

state:

[
    {
        "info": {
            "id": 253941,
            "location": "s3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2024-05-11/",
            "name": "dt",
            "value": "2024-05-11"
        },
        "state": {
            "arrival": {
                "athena_discovery": true,
                "first_data": "2024-05-13 12:48:56",
                "last_data": "2024-05-13 12:48:56"
            },
            "parquet": {
                "first_performed": null,
                "in_progress": null,
                "job_id": null,
                "last_performed": null,
                "queued": false
            }
        }
    },
    {
        "info": {
            "id": 253938,
            "location": "s3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2024-05-12/",
            "name": "dt",
            "value": "2024-05-12"
        },
        "state": {
            "arrival": {
                "athena_discovery": true,
                "first_data": "2024-05-13 12:48:36",
                "last_data": "2024-05-13 12:48:36"
            },
            "parquet": {
                "first_performed": null,
                "in_progress": null,
                "job_id": null,
                "last_performed": null,
                "queued": false
            }
        }
    },
    {
        "info": {
            "id": 254003,
            "location": "s3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2024-05-13/",
            "name": "dt",
            "value": "2024-05-13"
        },
        "state": {
            "arrival": {
                "athena_discovery": true,
                "first_data": "2024-05-14 03:38:28",
                "last_data": "2024-05-14 03:38:28"
            },
            "parquet": {
                "first_performed": null,
                "in_progress": null,
                "job_id": null,
                "last_performed": null,
                "queued": false
            }
        }
    },
    {
        "info": {
            "id": 259635,
            "location": "s3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2024-08-12/",
            "name": "dt",
            "value": "2024-08-12"
        },
        "state": {
            "arrival": {
                "athena_discovery": true,
                "first_data": "2024-08-14 09:49:34",
                "last_data": "2024-08-14 09:49:34"
            },
            "parquet": {
                "first_performed": null,
                "in_progress": null,
                "job_id": null,
                "last_performed": null,
                "queued": false
            }
        }
    },
    {
        "info": {
            "id": 259631,
            "location": "s3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2024-08-13/",
            "name": "dt",
            "value": "2024-08-13"
        },
        "state": {
            "arrival": {
                "athena_discovery": true,
                "first_data": "2024-08-14 09:48:55",
                "last_data": "2024-08-14 09:48:55"
            },
            "parquet": {
                "first_performed": null,
                "in_progress": null,
                "job_id": null,
                "last_performed": null,
                "queued": false
            }
        }
    }
]

type: string

Schema:

attributes:

{
    "_origin": "string",
    "agent_id": "string",
    "conversation_end": "string",
    "conversation_id": "string",
    "conversation_start": "string",
    "direction": "string",
    "disconnect_type": "string",
    "end_time": "string",
    "flow_id": "string",
    "ivr_id": "string",
    "origin_number": "string",
    "participant_id": "string",
    "participant_name": "string",
    "participant_type": "string",
    "queue_id": "string",
    "received_number": "string",
    "segment_type": "string",
    "sequence": "string",
    "start_time": "string",
    "wrapup_code": "string"
}

created: 2023-03-09 14:57:31

ddl:

{
    "create_database": "CREATE DATABASE IF NOT EXISTS `sfdl_dev_manual`",
    "create_table": "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_manual.genesys_inbound_call_segments`\n(\n`participant_type`STRING,`participant_name`STRING,`agent_id`STRING,`participant_id`STRING,`conversation_start`STRING,`disconnect_type`STRING,`wrapup_code`STRING,`_origin`STRING,`direction`STRING,`segment_type`STRING,`received_number`STRING,`sequence`STRING,`queue_id`STRING,`start_time`STRING,`ivr_id`STRING,`conversation_end`STRING,`end_time`STRING,`origin_number`STRING,`flow_id`STRING,`conversation_id`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments'\n",
    "drop_table": "DROP TABLE IF EXISTS `sfdl_dev_manual.genesys_inbound_call_segments`",
    "repair_table": "MSCK REPAIR TABLE sfdl_dev_manual.genesys_inbound_call_segments"
}

Server:

hostname: schemamanager-5447774fc6-4w58v
name: schema_manager
process_start: 2026-04-02 21:43:44
sf_env: dev
version: 2427328685

Raw:

{
    "name": {
        "created": "2019-08-21 16:46:06",
        "disabled": false,
        "format": "raw",
        "id": 1288,
        "is_partitioned": true,
        "is_schema_datatyped": false,
        "name": {
            "database": "sfdl_dev_manual",
            "table": "genesys_inbound_call_segments"
        },
        "type": "manual"
    },
    "partition": {
        "count": 5,
        "ddl": [
            "ALTER TABLE\n    `sfdl_dev_manual.genesys_inbound_call_segments`\nADD\n\n\n PARTITION (dt='2024-05-11') location 's3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2024-05-11/'\n PARTITION (dt='2024-05-12') location 's3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2024-05-12/'\n PARTITION (dt='2024-05-13') location 's3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2024-05-13/'\n PARTITION (dt='2024-08-12') location 's3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2024-08-12/'\n PARTITION (dt='2024-08-13') location 's3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2024-08-13/'\n;"
        ],
        "name": "dt",
        "state": [
            {
                "info": {
                    "id": 253941,
                    "location": "s3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2024-05-11/",
                    "name": "dt",
                    "value": "2024-05-11"
                },
                "state": {
                    "arrival": {
                        "athena_discovery": true,
                        "first_data": "2024-05-13 12:48:56",
                        "last_data": "2024-05-13 12:48:56"
                    },
                    "parquet": {
                        "first_performed": null,
                        "in_progress": null,
                        "job_id": null,
                        "last_performed": null,
                        "queued": false
                    }
                }
            },
            {
                "info": {
                    "id": 253938,
                    "location": "s3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2024-05-12/",
                    "name": "dt",
                    "value": "2024-05-12"
                },
                "state": {
                    "arrival": {
                        "athena_discovery": true,
                        "first_data": "2024-05-13 12:48:36",
                        "last_data": "2024-05-13 12:48:36"
                    },
                    "parquet": {
                        "first_performed": null,
                        "in_progress": null,
                        "job_id": null,
                        "last_performed": null,
                        "queued": false
                    }
                }
            },
            {
                "info": {
                    "id": 254003,
                    "location": "s3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2024-05-13/",
                    "name": "dt",
                    "value": "2024-05-13"
                },
                "state": {
                    "arrival": {
                        "athena_discovery": true,
                        "first_data": "2024-05-14 03:38:28",
                        "last_data": "2024-05-14 03:38:28"
                    },
                    "parquet": {
                        "first_performed": null,
                        "in_progress": null,
                        "job_id": null,
                        "last_performed": null,
                        "queued": false
                    }
                }
            },
            {
                "info": {
                    "id": 259635,
                    "location": "s3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2024-08-12/",
                    "name": "dt",
                    "value": "2024-08-12"
                },
                "state": {
                    "arrival": {
                        "athena_discovery": true,
                        "first_data": "2024-08-14 09:49:34",
                        "last_data": "2024-08-14 09:49:34"
                    },
                    "parquet": {
                        "first_performed": null,
                        "in_progress": null,
                        "job_id": null,
                        "last_performed": null,
                        "queued": false
                    }
                }
            },
            {
                "info": {
                    "id": 259631,
                    "location": "s3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments/dt=2024-08-13/",
                    "name": "dt",
                    "value": "2024-08-13"
                },
                "state": {
                    "arrival": {
                        "athena_discovery": true,
                        "first_data": "2024-08-14 09:48:55",
                        "last_data": "2024-08-14 09:48:55"
                    },
                    "parquet": {
                        "first_performed": null,
                        "in_progress": null,
                        "job_id": null,
                        "last_performed": null,
                        "queued": false
                    }
                }
            }
        ],
        "type": "string"
    },
    "schema": {
        "attributes": {
            "_origin": "string",
            "agent_id": "string",
            "conversation_end": "string",
            "conversation_id": "string",
            "conversation_start": "string",
            "direction": "string",
            "disconnect_type": "string",
            "end_time": "string",
            "flow_id": "string",
            "ivr_id": "string",
            "origin_number": "string",
            "participant_id": "string",
            "participant_name": "string",
            "participant_type": "string",
            "queue_id": "string",
            "received_number": "string",
            "segment_type": "string",
            "sequence": "string",
            "start_time": "string",
            "wrapup_code": "string"
        },
        "created": "2023-03-09 14:57:31",
        "ddl": {
            "create_database": "CREATE DATABASE IF NOT EXISTS `sfdl_dev_manual`",
            "create_table": "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_manual.genesys_inbound_call_segments`\n(\n`participant_type`STRING,`participant_name`STRING,`agent_id`STRING,`participant_id`STRING,`conversation_start`STRING,`disconnect_type`STRING,`wrapup_code`STRING,`_origin`STRING,`direction`STRING,`segment_type`STRING,`received_number`STRING,`sequence`STRING,`queue_id`STRING,`start_time`STRING,`ivr_id`STRING,`conversation_end`STRING,`end_time`STRING,`origin_number`STRING,`flow_id`STRING,`conversation_id`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-manual-genesys-dev/separated_dt/inbound_call_segments'\n",
            "drop_table": "DROP TABLE IF EXISTS `sfdl_dev_manual.genesys_inbound_call_segments`",
            "repair_table": "MSCK REPAIR TABLE sfdl_dev_manual.genesys_inbound_call_segments"
        }
    },
    "server": {
        "hostname": "schemamanager-5447774fc6-4w58v",
        "name": "schema_manager",
        "process_start": "2026-04-02 21:43:44",
        "sf_env": "dev",
        "version": "2427328685"
    }
}