Schema Info

Schema Name: sfdl_dev_drivethru.newbookdev_newbooknotifications

created: 2024-10-08 18:17:20
disabled: False
format: raw
id: 2889
is_partitioned: False
is_schema_datatyped: True

name:

{
    "database": "sfdl_dev_drivethru",
    "table": "newbookdev_newbooknotifications"
}

type: drivethrucsv

Schema Versions: 15

Version 4749153

created: 2024-11-05 23:23:58

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbooknotifications`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbooknotifications`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `period_from`  string , `period_to`  string , `type`  string , `for`  string , `for_id`  string , `message`  string , `all_staff`  string , `dismissable`  string , `display_on`  string , `ewn_alert_id`  string , `generated_by`  string , `generated_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-05/1730866568'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbooknotifications"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-05/1730866568
id: 4749153
processed: 1

raw_data:

{
    "all_staff": 9,
    "dismissable": 10,
    "display_on": 11,
    "ewn_alert_id": 12,
    "for": 6,
    "for_id": 7,
    "generated_by": 13,
    "generated_when": 14,
    "id": 2,
    "message": 8,
    "newbook_region": 0,
    "period_from": 3,
    "period_to": 4,
    "tenant": 1,
    "type": 5
}

schema_attributes:

{
    "all_staff": "string",
    "dismissable": "string",
    "display_on": "string",
    "ewn_alert_id": "string",
    "for": "string",
    "for_id": "string",
    "generated_by": "string",
    "generated_when": "string",
    "id": "string",
    "message": "string",
    "newbook_region": "string",
    "period_from": "string",
    "period_to": "string",
    "tenant": "string",
    "type": "string"
}

schema_name_id: 2889
schema_scan_id: 6675185
updated: 2024-11-05 23:24:10

Version 4748780

created: 2024-11-05 23:01:04

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbooknotifications`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbooknotifications`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `period_from`  string , `period_to`  string , `type`  string , `for`  string , `for_id`  string , `message`  string , `all_staff`  string , `dismissable`  string , `display_on`  string , `ewn_alert_id`  string , `generated_by`  string , `generated_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-05/1730866510'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbooknotifications"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-05/1730866510
id: 4748780
processed: 1

raw_data:

{
    "all_staff": 9,
    "dismissable": 10,
    "display_on": 11,
    "ewn_alert_id": 12,
    "for": 6,
    "for_id": 7,
    "generated_by": 13,
    "generated_when": 14,
    "id": 2,
    "message": 8,
    "newbook_region": 0,
    "period_from": 3,
    "period_to": 4,
    "tenant": 1,
    "type": 5
}

schema_attributes:

{
    "all_staff": "string",
    "dismissable": "string",
    "display_on": "string",
    "ewn_alert_id": "string",
    "for": "string",
    "for_id": "string",
    "generated_by": "string",
    "generated_when": "string",
    "id": "string",
    "message": "string",
    "newbook_region": "string",
    "period_from": "string",
    "period_to": "string",
    "tenant": "string",
    "type": "string"
}

schema_name_id: 2889
schema_scan_id: 6674812
updated: 2024-11-05 23:01:17

Version 4748115

created: 2024-11-05 22:35:21

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbooknotifications`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbooknotifications`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `period_from`  string , `period_to`  string , `type`  string , `for`  string , `for_id`  string , `message`  string , `all_staff`  string , `dismissable`  string , `display_on`  string , `ewn_alert_id`  string , `generated_by`  string , `generated_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-05/1730866379'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbooknotifications"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-05/1730866379
id: 4748115
processed: 1

raw_data:

{
    "all_staff": 9,
    "dismissable": 10,
    "display_on": 11,
    "ewn_alert_id": 12,
    "for": 6,
    "for_id": 7,
    "generated_by": 13,
    "generated_when": 14,
    "id": 2,
    "message": 8,
    "newbook_region": 0,
    "period_from": 3,
    "period_to": 4,
    "tenant": 1,
    "type": 5
}

schema_attributes:

{
    "all_staff": "string",
    "dismissable": "string",
    "display_on": "string",
    "ewn_alert_id": "string",
    "for": "string",
    "for_id": "string",
    "generated_by": "string",
    "generated_when": "string",
    "id": "string",
    "message": "string",
    "newbook_region": "string",
    "period_from": "string",
    "period_to": "string",
    "tenant": "string",
    "type": "string"
}

schema_name_id: 2889
schema_scan_id: 6674147
updated: 2024-11-05 22:35:35

Version 4747212

created: 2024-11-05 05:02:28

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbooknotifications`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbooknotifications`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `period_from`  string , `period_to`  string , `type`  string , `for`  string , `for_id`  string , `message`  string , `all_staff`  string , `dismissable`  string , `display_on`  string , `ewn_alert_id`  string , `generated_by`  string , `generated_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730785767'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbooknotifications"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730785767
id: 4747212
processed: 1

raw_data:

{
    "all_staff": 9,
    "dismissable": 10,
    "display_on": 11,
    "ewn_alert_id": 12,
    "for": 6,
    "for_id": 7,
    "generated_by": 13,
    "generated_when": 14,
    "id": 2,
    "message": 8,
    "newbook_region": 0,
    "period_from": 3,
    "period_to": 4,
    "tenant": 1,
    "type": 5
}

schema_attributes:

{
    "all_staff": "string",
    "dismissable": "string",
    "display_on": "string",
    "ewn_alert_id": "string",
    "for": "string",
    "for_id": "string",
    "generated_by": "string",
    "generated_when": "string",
    "id": "string",
    "message": "string",
    "newbook_region": "string",
    "period_from": "string",
    "period_to": "string",
    "tenant": "string",
    "type": "string"
}

schema_name_id: 2889
schema_scan_id: 6672446
updated: 2024-11-05 05:02:42

Version 4746399

created: 2024-11-05 04:01:19

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbooknotifications`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbooknotifications`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `period_from`  string , `period_to`  string , `type`  string , `for`  string , `for_id`  string , `message`  string , `all_staff`  string , `dismissable`  string , `display_on`  string , `ewn_alert_id`  string , `generated_by`  string , `generated_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730785150'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbooknotifications"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730785150
id: 4746399
processed: 1

raw_data:

{
    "all_staff": 9,
    "dismissable": 10,
    "display_on": 11,
    "ewn_alert_id": 12,
    "for": 6,
    "for_id": 7,
    "generated_by": 13,
    "generated_when": 14,
    "id": 2,
    "message": 8,
    "newbook_region": 0,
    "period_from": 3,
    "period_to": 4,
    "tenant": 1,
    "type": 5
}

schema_attributes:

{
    "all_staff": "string",
    "dismissable": "string",
    "display_on": "string",
    "ewn_alert_id": "string",
    "for": "string",
    "for_id": "string",
    "generated_by": "string",
    "generated_when": "string",
    "id": "string",
    "message": "string",
    "newbook_region": "string",
    "period_from": "string",
    "period_to": "string",
    "tenant": "string",
    "type": "string"
}

schema_name_id: 2889
schema_scan_id: 6671620
updated: 2024-11-05 04:01:30

Version 4745544

created: 2024-11-05 03:32:15

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbooknotifications`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbooknotifications`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `period_from`  string , `period_to`  string , `type`  string , `for`  string , `for_id`  string , `message`  string , `all_staff`  string , `dismissable`  string , `display_on`  string , `ewn_alert_id`  string , `generated_by`  string , `generated_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730784845'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbooknotifications"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730784845
id: 4745544
processed: 1

raw_data:

{
    "all_staff": 9,
    "dismissable": 10,
    "display_on": 11,
    "ewn_alert_id": 12,
    "for": 6,
    "for_id": 7,
    "generated_by": 13,
    "generated_when": 14,
    "id": 2,
    "message": 8,
    "newbook_region": 0,
    "period_from": 3,
    "period_to": 4,
    "tenant": 1,
    "type": 5
}

schema_attributes:

{
    "all_staff": "string",
    "dismissable": "string",
    "display_on": "string",
    "ewn_alert_id": "string",
    "for": "string",
    "for_id": "string",
    "generated_by": "string",
    "generated_when": "string",
    "id": "string",
    "message": "string",
    "newbook_region": "string",
    "period_from": "string",
    "period_to": "string",
    "tenant": "string",
    "type": "string"
}

schema_name_id: 2889
schema_scan_id: 6670761
updated: 2024-11-05 03:32:28

Version 4745407

created: 2024-11-05 03:28:36

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbooknotifications`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbooknotifications`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `period_from`  string , `period_to`  string , `type`  string , `for`  string , `for_id`  string , `message`  string , `all_staff`  string , `dismissable`  string , `display_on`  string , `ewn_alert_id`  string , `generated_by`  string , `generated_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730784242'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbooknotifications"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730784242
id: 4745407
processed: 1

raw_data:

{
    "all_staff": 9,
    "dismissable": 10,
    "display_on": 11,
    "ewn_alert_id": 12,
    "for": 6,
    "for_id": 7,
    "generated_by": 13,
    "generated_when": 14,
    "id": 2,
    "message": 8,
    "newbook_region": 0,
    "period_from": 3,
    "period_to": 4,
    "tenant": 1,
    "type": 5
}

schema_attributes:

{
    "all_staff": "string",
    "dismissable": "string",
    "display_on": "string",
    "ewn_alert_id": "string",
    "for": "string",
    "for_id": "string",
    "generated_by": "string",
    "generated_when": "string",
    "id": "string",
    "message": "string",
    "newbook_region": "string",
    "period_from": "string",
    "period_to": "string",
    "tenant": "string",
    "type": "string"
}

schema_name_id: 2889
schema_scan_id: 6670624
updated: 2024-11-05 03:28:47

Version 4743659

created: 2024-11-05 02:42:29

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbooknotifications`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbooknotifications`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `period_from`  string , `period_to`  string , `type`  string , `for`  string , `for_id`  string , `message`  string , `all_staff`  string , `dismissable`  string , `display_on`  string , `ewn_alert_id`  string , `generated_by`  string , `generated_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730783918'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbooknotifications"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730783918
id: 4743659
processed: 1

raw_data:

{
    "all_staff": 9,
    "dismissable": 10,
    "display_on": 11,
    "ewn_alert_id": 12,
    "for": 6,
    "for_id": 7,
    "generated_by": 13,
    "generated_when": 14,
    "id": 2,
    "message": 8,
    "newbook_region": 0,
    "period_from": 3,
    "period_to": 4,
    "tenant": 1,
    "type": 5
}

schema_attributes:

{
    "all_staff": "string",
    "dismissable": "string",
    "display_on": "string",
    "ewn_alert_id": "string",
    "for": "string",
    "for_id": "string",
    "generated_by": "string",
    "generated_when": "string",
    "id": "string",
    "message": "string",
    "newbook_region": "string",
    "period_from": "string",
    "period_to": "string",
    "tenant": "string",
    "type": "string"
}

schema_name_id: 2889
schema_scan_id: 6668876
updated: 2024-11-05 02:42:40

Version 4743603

created: 2024-11-05 02:40:57

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbooknotifications`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbooknotifications`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `period_from`  string , `period_to`  string , `type`  string , `for`  string , `for_id`  string , `message`  string , `all_staff`  string , `dismissable`  string , `display_on`  string , `ewn_alert_id`  string , `generated_by`  string , `generated_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730784079'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbooknotifications"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730784079
id: 4743603
processed: 1

raw_data:

{
    "all_staff": 9,
    "dismissable": 10,
    "display_on": 11,
    "ewn_alert_id": 12,
    "for": 6,
    "for_id": 7,
    "generated_by": 13,
    "generated_when": 14,
    "id": 2,
    "message": 8,
    "newbook_region": 0,
    "period_from": 3,
    "period_to": 4,
    "tenant": 1,
    "type": 5
}

schema_attributes:

{
    "all_staff": "string",
    "dismissable": "string",
    "display_on": "string",
    "ewn_alert_id": "string",
    "for": "string",
    "for_id": "string",
    "generated_by": "string",
    "generated_when": "string",
    "id": "string",
    "message": "string",
    "newbook_region": "string",
    "period_from": "string",
    "period_to": "string",
    "tenant": "string",
    "type": "string"
}

schema_name_id: 2889
schema_scan_id: 6668820
updated: 2024-11-05 02:41:08

Version 4742621

created: 2024-11-05 02:15:02

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbooknotifications`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbooknotifications`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `period_from`  string , `period_to`  string , `type`  string , `for`  string , `for_id`  string , `message`  string , `all_staff`  string , `dismissable`  string , `display_on`  string , `ewn_alert_id`  string , `generated_by`  string , `generated_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730783793'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbooknotifications"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730783793
id: 4742621
processed: 1

raw_data:

{
    "all_staff": 9,
    "dismissable": 10,
    "display_on": 11,
    "ewn_alert_id": 12,
    "for": 6,
    "for_id": 7,
    "generated_by": 13,
    "generated_when": 14,
    "id": 2,
    "message": 8,
    "newbook_region": 0,
    "period_from": 3,
    "period_to": 4,
    "tenant": 1,
    "type": 5
}

schema_attributes:

{
    "all_staff": "string",
    "dismissable": "string",
    "display_on": "string",
    "ewn_alert_id": "string",
    "for": "string",
    "for_id": "string",
    "generated_by": "string",
    "generated_when": "string",
    "id": "string",
    "message": "string",
    "newbook_region": "string",
    "period_from": "string",
    "period_to": "string",
    "tenant": "string",
    "type": "string"
}

schema_name_id: 2889
schema_scan_id: 6667838
updated: 2024-11-05 02:15:13

Version 4742108

created: 2024-11-05 02:01:16

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbooknotifications`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbooknotifications`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `period_from`  string , `period_to`  string , `type`  string , `for`  string , `for_id`  string , `message`  string , `all_staff`  string , `dismissable`  string , `display_on`  string , `ewn_alert_id`  string , `generated_by`  string , `generated_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730783684'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbooknotifications"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730783684
id: 4742108
processed: 1

raw_data:

{
    "all_staff": 9,
    "dismissable": 10,
    "display_on": 11,
    "ewn_alert_id": 12,
    "for": 6,
    "for_id": 7,
    "generated_by": 13,
    "generated_when": 14,
    "id": 2,
    "message": 8,
    "newbook_region": 0,
    "period_from": 3,
    "period_to": 4,
    "tenant": 1,
    "type": 5
}

schema_attributes:

{
    "all_staff": "string",
    "dismissable": "string",
    "display_on": "string",
    "ewn_alert_id": "string",
    "for": "string",
    "for_id": "string",
    "generated_by": "string",
    "generated_when": "string",
    "id": "string",
    "message": "string",
    "newbook_region": "string",
    "period_from": "string",
    "period_to": "string",
    "tenant": "string",
    "type": "string"
}

schema_name_id: 2889
schema_scan_id: 6667325
updated: 2024-11-05 02:01:29

Version 4740807

created: 2024-11-05 01:27:00

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbooknotifications`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbooknotifications`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `period_from`  string , `period_to`  string , `type`  string , `for`  string , `for_id`  string , `message`  string , `all_staff`  string , `dismissable`  string , `display_on`  string , `ewn_alert_id`  string , `generated_by`  string , `generated_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730783550'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbooknotifications"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730783550
id: 4740807
processed: 1

raw_data:

{
    "all_staff": 9,
    "dismissable": 10,
    "display_on": 11,
    "ewn_alert_id": 12,
    "for": 6,
    "for_id": 7,
    "generated_by": 13,
    "generated_when": 14,
    "id": 2,
    "message": 8,
    "newbook_region": 0,
    "period_from": 3,
    "period_to": 4,
    "tenant": 1,
    "type": 5
}

schema_attributes:

{
    "all_staff": "string",
    "dismissable": "string",
    "display_on": "string",
    "ewn_alert_id": "string",
    "for": "string",
    "for_id": "string",
    "generated_by": "string",
    "generated_when": "string",
    "id": "string",
    "message": "string",
    "newbook_region": "string",
    "period_from": "string",
    "period_to": "string",
    "tenant": "string",
    "type": "string"
}

schema_name_id: 2889
schema_scan_id: 6666024
updated: 2024-11-05 01:27:11

Version 4740478

created: 2024-11-05 01:18:23

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbooknotifications`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbooknotifications`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `period_from`  string , `period_to`  string , `type`  string , `for`  string , `for_id`  string , `message`  string , `all_staff`  string , `dismissable`  string , `display_on`  string , `ewn_alert_id`  string , `generated_by`  string , `generated_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730783331'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbooknotifications"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730783331
id: 4740478
processed: 1

raw_data:

{
    "all_staff": 9,
    "dismissable": 10,
    "display_on": 11,
    "ewn_alert_id": 12,
    "for": 6,
    "for_id": 7,
    "generated_by": 13,
    "generated_when": 14,
    "id": 2,
    "message": 8,
    "newbook_region": 0,
    "period_from": 3,
    "period_to": 4,
    "tenant": 1,
    "type": 5
}

schema_attributes:

{
    "all_staff": "string",
    "dismissable": "string",
    "display_on": "string",
    "ewn_alert_id": "string",
    "for": "string",
    "for_id": "string",
    "generated_by": "string",
    "generated_when": "string",
    "id": "string",
    "message": "string",
    "newbook_region": "string",
    "period_from": "string",
    "period_to": "string",
    "tenant": "string",
    "type": "string"
}

schema_name_id: 2889
schema_scan_id: 6665695
updated: 2024-11-05 01:18:34

Version 4739748

created: 2024-11-05 00:58:50

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbooknotifications`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbooknotifications`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `period_from`  string , `period_to`  string , `type`  string , `for`  string , `for_id`  string , `message`  string , `all_staff`  string , `dismissable`  string , `display_on`  string , `ewn_alert_id`  string , `generated_by`  string , `generated_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730782931'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbooknotifications"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730782931
id: 4739748
processed: 1

raw_data:

{
    "all_staff": 9,
    "dismissable": 10,
    "display_on": 11,
    "ewn_alert_id": 12,
    "for": 6,
    "for_id": 7,
    "generated_by": 13,
    "generated_when": 14,
    "id": 2,
    "message": 8,
    "newbook_region": 0,
    "period_from": 3,
    "period_to": 4,
    "tenant": 1,
    "type": 5
}

schema_attributes:

{
    "all_staff": "string",
    "dismissable": "string",
    "display_on": "string",
    "ewn_alert_id": "string",
    "for": "string",
    "for_id": "string",
    "generated_by": "string",
    "generated_when": "string",
    "id": "string",
    "message": "string",
    "newbook_region": "string",
    "period_from": "string",
    "period_to": "string",
    "tenant": "string",
    "type": "string"
}

schema_name_id: 2889
schema_scan_id: 6664965
updated: 2024-11-05 00:59:03

Version 4738425

created: 2024-11-05 00:24:05

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbooknotifications`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbooknotifications`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `period_from`  string , `period_to`  string , `type`  string , `for`  string , `for_id`  string , `message`  string , `all_staff`  string , `dismissable`  string , `display_on`  string , `ewn_alert_id`  string , `generated_by`  string , `generated_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730782757'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbooknotifications"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730782757
id: 4738425
processed: 1

raw_data:

{
    "all_staff": 9,
    "dismissable": 10,
    "display_on": 11,
    "ewn_alert_id": 12,
    "for": 6,
    "for_id": 7,
    "generated_by": 13,
    "generated_when": 14,
    "id": 2,
    "message": 8,
    "newbook_region": 0,
    "period_from": 3,
    "period_to": 4,
    "tenant": 1,
    "type": 5
}

schema_attributes:

{
    "all_staff": "string",
    "dismissable": "string",
    "display_on": "string",
    "ewn_alert_id": "string",
    "for": "string",
    "for_id": "string",
    "generated_by": "string",
    "generated_when": "string",
    "id": "string",
    "message": "string",
    "newbook_region": "string",
    "period_from": "string",
    "period_to": "string",
    "tenant": "string",
    "type": "string"
}

schema_name_id: 2889
schema_scan_id: 6663634
updated: 2024-11-05 00:24:16

Schema Scans: 15

Last at 2024-11-05 23:23:58

Scan 6675185

duration: 0:00:11.844730

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6675185

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-05/1730866568",
        "key": "raw/newbookdev_newbooknotifications/2024-11-05/1730866568/notifications.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbooknotifications",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 223803660272869243,
        "service_name": "drive-thru",
        "span_id": 3860049947792608370,
        "trace_id": 8155962372359556619
    }
}

running: False
schema_name_id: 2889
start_time: 2024-11-05 23:23:58
success: True
trace_id: 8155962372359556619

Scan 6674812

duration: 0:00:14.293645

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6674812

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-05/1730866510",
        "key": "raw/newbookdev_newbooknotifications/2024-11-05/1730866510/notifications.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbooknotifications",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 2703723153164164158,
        "service_name": "drive-thru",
        "span_id": 11263026433629502916,
        "trace_id": 95345016107193771
    }
}

running: False
schema_name_id: 2889
start_time: 2024-11-05 23:01:03
success: True
trace_id: 95345016107193771

Scan 6674147

duration: 0:00:14.402781

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6674147

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-05/1730866379",
        "key": "raw/newbookdev_newbooknotifications/2024-11-05/1730866379/notifications.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbooknotifications",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 12936674088247055612,
        "service_name": "drive-thru",
        "span_id": 10270702976643109065,
        "trace_id": 6128218805730173740
    }
}

running: False
schema_name_id: 2889
start_time: 2024-11-05 22:35:21
success: True
trace_id: 6128218805730173740

Scan 6672446

duration: 0:00:14.364289

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6672446

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730785767",
        "key": "raw/newbookdev_newbooknotifications/2024-11-04/1730785767/notifications.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbooknotifications",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 6289514823469086536,
        "service_name": "drive-thru",
        "span_id": 7111055960265341547,
        "trace_id": 4918835645359794909
    }
}

running: False
schema_name_id: 2889
start_time: 2024-11-05 05:02:28
success: True
trace_id: 4918835645359794909

Scan 6671620

duration: 0:00:11.914409

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6671620

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730785150",
        "key": "raw/newbookdev_newbooknotifications/2024-11-04/1730785150/notifications.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbooknotifications",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 18420855749182133784,
        "service_name": "drive-thru",
        "span_id": 17887713149525406439,
        "trace_id": 3111001659556046820
    }
}

running: False
schema_name_id: 2889
start_time: 2024-11-05 04:01:19
success: True
trace_id: 3111001659556046820

Scan 6670761

duration: 0:00:14.073041

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6670761

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730784845",
        "key": "raw/newbookdev_newbooknotifications/2024-11-04/1730784845/notifications.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbooknotifications",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 11406438010012829687,
        "service_name": "drive-thru",
        "span_id": 15292302152454571830,
        "trace_id": 15287838607457698309
    }
}

running: False
schema_name_id: 2889
start_time: 2024-11-05 03:32:14
success: True
trace_id: 15287838607457698309

Scan 6670624

duration: 0:00:11.679864

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6670624

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730784242",
        "key": "raw/newbookdev_newbooknotifications/2024-11-04/1730784242/notifications.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbooknotifications",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 7264023246732612789,
        "service_name": "drive-thru",
        "span_id": 5621255837422316426,
        "trace_id": 3574803235938358382
    }
}

running: False
schema_name_id: 2889
start_time: 2024-11-05 03:28:35
success: True
trace_id: 3574803235938358382

Scan 6668876

duration: 0:00:11.602050

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6668876

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730783918",
        "key": "raw/newbookdev_newbooknotifications/2024-11-04/1730783918/notifications.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbooknotifications",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 15134641826000810986,
        "service_name": "drive-thru",
        "span_id": 7447198402602624559,
        "trace_id": 1259445609719581707
    }
}

running: False
schema_name_id: 2889
start_time: 2024-11-05 02:42:29
success: True
trace_id: 1259445609719581707

Scan 6668820

duration: 0:00:11.629712

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6668820

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730784079",
        "key": "raw/newbookdev_newbooknotifications/2024-11-04/1730784079/notifications.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbooknotifications",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 11071339592035773420,
        "service_name": "drive-thru",
        "span_id": 14153151656460655604,
        "trace_id": 12205358234408506794
    }
}

running: False
schema_name_id: 2889
start_time: 2024-11-05 02:40:56
success: True
trace_id: 12205358234408506794

Scan 6667838

duration: 0:00:11.772534

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6667838

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730783793",
        "key": "raw/newbookdev_newbooknotifications/2024-11-04/1730783793/notifications.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbooknotifications",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 8167417061267049195,
        "service_name": "drive-thru",
        "span_id": 15916886922924383860,
        "trace_id": 15279792864046887693
    }
}

running: False
schema_name_id: 2889
start_time: 2024-11-05 02:15:01
success: True
trace_id: 15279792864046887693

Scan 6667325

duration: 0:00:14.408417

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6667325

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730783684",
        "key": "raw/newbookdev_newbooknotifications/2024-11-04/1730783684/notifications.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbooknotifications",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 17356763959027303549,
        "service_name": "drive-thru",
        "span_id": 11214132819754259704,
        "trace_id": 5824789981483828931
    }
}

running: False
schema_name_id: 2889
start_time: 2024-11-05 02:01:15
success: True
trace_id: 5824789981483828931

Scan 6666024

duration: 0:00:11.989939

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6666024

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730783550",
        "key": "raw/newbookdev_newbooknotifications/2024-11-04/1730783550/notifications.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbooknotifications",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 14170778604002220143,
        "service_name": "drive-thru",
        "span_id": 9044829253251986851,
        "trace_id": 15810462685859049406
    }
}

running: False
schema_name_id: 2889
start_time: 2024-11-05 01:26:59
success: True
trace_id: 15810462685859049406

Scan 6665695

duration: 0:00:11.667759

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6665695

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730783331",
        "key": "raw/newbookdev_newbooknotifications/2024-11-04/1730783331/notifications.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbooknotifications",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 5900791334138904728,
        "service_name": "drive-thru",
        "span_id": 17757062814034592262,
        "trace_id": 9411853222930990109
    }
}

running: False
schema_name_id: 2889
start_time: 2024-11-05 01:18:22
success: True
trace_id: 9411853222930990109

Scan 6664965

duration: 0:00:14.348282

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6664965

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730782931",
        "key": "raw/newbookdev_newbooknotifications/2024-11-04/1730782931/notifications.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbooknotifications",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 3295266294572338119,
        "service_name": "drive-thru",
        "span_id": 10799371763053966791,
        "trace_id": 15855628869346956330
    }
}

running: False
schema_name_id: 2889
start_time: 2024-11-05 00:58:49
success: True
trace_id: 15855628869346956330

Scan 6663634

duration: 0:00:11.619084

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6663634

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-04/1730782757",
        "key": "raw/newbookdev_newbooknotifications/2024-11-04/1730782757/notifications.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbooknotifications",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 9474195749876185321,
        "service_name": "drive-thru",
        "span_id": 479065034766571816,
        "trace_id": 3012510404878604141
    }
}

running: False
schema_name_id: 2889
start_time: 2024-11-05 00:24:05
success: True
trace_id: 3012510404878604141

Partitions:

count: 0
ddl: None
name: None
state:
```
[]
```
type: None

Schema:

attributes:

{
    "all_staff": "string",
    "dismissable": "string",
    "display_on": "string",
    "ewn_alert_id": "string",
    "for": "string",
    "for_id": "string",
    "generated_by": "string",
    "generated_when": "string",
    "id": "string",
    "message": "string",
    "newbook_region": "string",
    "period_from": "string",
    "period_to": "string",
    "tenant": "string",
    "type": "string"
}

created: 2024-11-05 23:23:58

ddl:

{
    "create_database": "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "create_table": "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbooknotifications`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `period_from`  string , `period_to`  string , `type`  string , `for`  string , `for_id`  string , `message`  string , `all_staff`  string , `dismissable`  string , `display_on`  string , `ewn_alert_id`  string , `generated_by`  string , `generated_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-05/1730866568'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "drop_table": "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbooknotifications`",
    "repair_table": "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbooknotifications"
}

Server:

hostname: schemamanager-5447774fc6-4w58v
name: schema_manager
process_start: 2026-04-02 21:43:44
sf_env: dev
version: 2427328685

Raw:

{
    "name": {
        "created": "2024-10-08 18:17:20",
        "disabled": false,
        "format": "raw",
        "id": 2889,
        "is_partitioned": false,
        "is_schema_datatyped": true,
        "name": {
            "database": "sfdl_dev_drivethru",
            "table": "newbookdev_newbooknotifications"
        },
        "type": "drivethrucsv"
    },
    "partition": {
        "count": 0,
        "ddl": null,
        "name": null,
        "state": [],
        "type": null
    },
    "schema": {
        "attributes": {
            "all_staff": "string",
            "dismissable": "string",
            "display_on": "string",
            "ewn_alert_id": "string",
            "for": "string",
            "for_id": "string",
            "generated_by": "string",
            "generated_when": "string",
            "id": "string",
            "message": "string",
            "newbook_region": "string",
            "period_from": "string",
            "period_to": "string",
            "tenant": "string",
            "type": "string"
        },
        "created": "2024-11-05 23:23:58",
        "ddl": {
            "create_database": "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
            "create_table": "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbooknotifications`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `period_from`  string , `period_to`  string , `type`  string , `for`  string , `for_id`  string , `message`  string , `all_staff`  string , `dismissable`  string , `display_on`  string , `ewn_alert_id`  string , `generated_by`  string , `generated_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbooknotifications/2024-11-05/1730866568'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
            "drop_table": "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbooknotifications`",
            "repair_table": "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbooknotifications"
        }
    },
    "server": {
        "hostname": "schemamanager-5447774fc6-4w58v",
        "name": "schema_manager",
        "process_start": "2026-04-02 21:43:44",
        "sf_env": "dev",
        "version": "2427328685"
    }
}