Schema Info

Schema Name: sfdl_dev_drivethru.newbookdev_newbookinsurance_policies

created: 2024-10-08 17:55:55
disabled: False
format: raw
id: 2671
is_partitioned: False
is_schema_datatyped: True

name:

{
    "database": "sfdl_dev_drivethru",
    "table": "newbookdev_newbookinsurance_policies"
}

type: drivethrucsv

Schema Versions: 15

Version 4749057

created: 2024-11-05 23:17:09

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `for`  string , `for_id`  string , `company`  string , `policy_type`  string , `policy_number`  string , `expiry_date`  string , `added_by`  string , `added_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-05/1730866516'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbookinsurance_policies"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-05/1730866516
id: 4749057
processed: 1

raw_data:

{
    "added_by": 9,
    "added_when": 10,
    "company": 5,
    "expiry_date": 8,
    "for": 3,
    "for_id": 4,
    "id": 2,
    "newbook_region": 0,
    "policy_number": 7,
    "policy_type": 6,
    "tenant": 1
}

schema_attributes:

{
    "added_by": "string",
    "added_when": "string",
    "company": "string",
    "expiry_date": "string",
    "for": "string",
    "for_id": "string",
    "id": "string",
    "newbook_region": "string",
    "policy_number": "string",
    "policy_type": "string",
    "tenant": "string"
}

schema_name_id: 2671
schema_scan_id: 6675089
updated: 2024-11-05 23:17:20

Version 4748546

created: 2024-11-05 22:50:36

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `for`  string , `for_id`  string , `company`  string , `policy_type`  string , `policy_number`  string , `expiry_date`  string , `added_by`  string , `added_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-05/1730866443'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbookinsurance_policies"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-05/1730866443
id: 4748546
processed: 1

raw_data:

{
    "added_by": 9,
    "added_when": 10,
    "company": 5,
    "expiry_date": 8,
    "for": 3,
    "for_id": 4,
    "id": 2,
    "newbook_region": 0,
    "policy_number": 7,
    "policy_type": 6,
    "tenant": 1
}

schema_attributes:

{
    "added_by": "string",
    "added_when": "string",
    "company": "string",
    "expiry_date": "string",
    "for": "string",
    "for_id": "string",
    "id": "string",
    "newbook_region": "string",
    "policy_number": "string",
    "policy_type": "string",
    "tenant": "string"
}

schema_name_id: 2671
schema_scan_id: 6674578
updated: 2024-11-05 22:50:49

Version 4747844

created: 2024-11-05 22:27:29

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `for`  string , `for_id`  string , `company`  string , `policy_type`  string , `policy_number`  string , `expiry_date`  string , `added_by`  string , `added_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-05/1730866319'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbookinsurance_policies"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-05/1730866319
id: 4747844
processed: 1

raw_data:

{
    "added_by": 9,
    "added_when": 10,
    "company": 5,
    "expiry_date": 8,
    "for": 3,
    "for_id": 4,
    "id": 2,
    "newbook_region": 0,
    "policy_number": 7,
    "policy_type": 6,
    "tenant": 1
}

schema_attributes:

{
    "added_by": "string",
    "added_when": "string",
    "company": "string",
    "expiry_date": "string",
    "for": "string",
    "for_id": "string",
    "id": "string",
    "newbook_region": "string",
    "policy_number": "string",
    "policy_type": "string",
    "tenant": "string"
}

schema_name_id: 2671
schema_scan_id: 6673876
updated: 2024-11-05 22:27:43

Version 4746879

created: 2024-11-05 04:29:51

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `for`  string , `for_id`  string , `company`  string , `policy_type`  string , `policy_number`  string , `expiry_date`  string , `added_by`  string , `added_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730785727'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbookinsurance_policies"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730785727
id: 4746879
processed: 1

raw_data:

{
    "added_by": 9,
    "added_when": 10,
    "company": 5,
    "expiry_date": 8,
    "for": 3,
    "for_id": 4,
    "id": 2,
    "newbook_region": 0,
    "policy_number": 7,
    "policy_type": 6,
    "tenant": 1
}

schema_attributes:

{
    "added_by": "string",
    "added_when": "string",
    "company": "string",
    "expiry_date": "string",
    "for": "string",
    "for_id": "string",
    "id": "string",
    "newbook_region": "string",
    "policy_number": "string",
    "policy_type": "string",
    "tenant": "string"
}

schema_name_id: 2671
schema_scan_id: 6672109
updated: 2024-11-05 04:30:02

Version 4745488

created: 2024-11-05 03:30:46

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `for`  string , `for_id`  string , `company`  string , `policy_type`  string , `policy_number`  string , `expiry_date`  string , `added_by`  string , `added_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730785128'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbookinsurance_policies"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730785128
id: 4745488
processed: 1

raw_data:

{
    "added_by": 9,
    "added_when": 10,
    "company": 5,
    "expiry_date": 8,
    "for": 3,
    "for_id": 4,
    "id": 2,
    "newbook_region": 0,
    "policy_number": 7,
    "policy_type": 6,
    "tenant": 1
}

schema_attributes:

{
    "added_by": "string",
    "added_when": "string",
    "company": "string",
    "expiry_date": "string",
    "for": "string",
    "for_id": "string",
    "id": "string",
    "newbook_region": "string",
    "policy_number": "string",
    "policy_type": "string",
    "tenant": "string"
}

schema_name_id: 2671
schema_scan_id: 6670705
updated: 2024-11-05 03:31:00

Version 4745466

created: 2024-11-05 03:30:09

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `for`  string , `for_id`  string , `company`  string , `policy_type`  string , `policy_number`  string , `expiry_date`  string , `added_by`  string , `added_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730784823'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbookinsurance_policies"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730784823
id: 4745466
processed: 1

raw_data:

{
    "added_by": 9,
    "added_when": 10,
    "company": 5,
    "expiry_date": 8,
    "for": 3,
    "for_id": 4,
    "id": 2,
    "newbook_region": 0,
    "policy_number": 7,
    "policy_type": 6,
    "tenant": 1
}

schema_attributes:

{
    "added_by": "string",
    "added_when": "string",
    "company": "string",
    "expiry_date": "string",
    "for": "string",
    "for_id": "string",
    "id": "string",
    "newbook_region": "string",
    "policy_number": "string",
    "policy_type": "string",
    "tenant": "string"
}

schema_name_id: 2671
schema_scan_id: 6670683
updated: 2024-11-05 03:30:20

Version 4744623

created: 2024-11-05 03:08:00

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `for`  string , `for_id`  string , `company`  string , `policy_type`  string , `policy_number`  string , `expiry_date`  string , `added_by`  string , `added_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730784219'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbookinsurance_policies"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730784219
id: 4744623
processed: 1

raw_data:

{
    "added_by": 9,
    "added_when": 10,
    "company": 5,
    "expiry_date": 8,
    "for": 3,
    "for_id": 4,
    "id": 2,
    "newbook_region": 0,
    "policy_number": 7,
    "policy_type": 6,
    "tenant": 1
}

schema_attributes:

{
    "added_by": "string",
    "added_when": "string",
    "company": "string",
    "expiry_date": "string",
    "for": "string",
    "for_id": "string",
    "id": "string",
    "newbook_region": "string",
    "policy_number": "string",
    "policy_type": "string",
    "tenant": "string"
}

schema_name_id: 2671
schema_scan_id: 6669840
updated: 2024-11-05 03:08:11

Version 4744015

created: 2024-11-05 02:51:49

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `for`  string , `for_id`  string , `company`  string , `policy_type`  string , `policy_number`  string , `expiry_date`  string , `added_by`  string , `added_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730784057'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbookinsurance_policies"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730784057
id: 4744015
processed: 1

raw_data:

{
    "added_by": 9,
    "added_when": 10,
    "company": 5,
    "expiry_date": 8,
    "for": 3,
    "for_id": 4,
    "id": 2,
    "newbook_region": 0,
    "policy_number": 7,
    "policy_type": 6,
    "tenant": 1
}

schema_attributes:

{
    "added_by": "string",
    "added_when": "string",
    "company": "string",
    "expiry_date": "string",
    "for": "string",
    "for_id": "string",
    "id": "string",
    "newbook_region": "string",
    "policy_number": "string",
    "policy_type": "string",
    "tenant": "string"
}

schema_name_id: 2671
schema_scan_id: 6669232
updated: 2024-11-05 02:52:00

Version 4743378

created: 2024-11-05 02:35:01

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `for`  string , `for_id`  string , `company`  string , `policy_type`  string , `policy_number`  string , `expiry_date`  string , `added_by`  string , `added_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730783901'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbookinsurance_policies"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730783901
id: 4743378
processed: 1

raw_data:

{
    "added_by": 9,
    "added_when": 10,
    "company": 5,
    "expiry_date": 8,
    "for": 3,
    "for_id": 4,
    "id": 2,
    "newbook_region": 0,
    "policy_number": 7,
    "policy_type": 6,
    "tenant": 1
}

schema_attributes:

{
    "added_by": "string",
    "added_when": "string",
    "company": "string",
    "expiry_date": "string",
    "for": "string",
    "for_id": "string",
    "id": "string",
    "newbook_region": "string",
    "policy_number": "string",
    "policy_type": "string",
    "tenant": "string"
}

schema_name_id: 2671
schema_scan_id: 6668595
updated: 2024-11-05 02:35:12

Version 4742663

created: 2024-11-05 02:16:08

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `for`  string , `for_id`  string , `company`  string , `policy_type`  string , `policy_number`  string , `expiry_date`  string , `added_by`  string , `added_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730783771'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbookinsurance_policies"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730783771
id: 4742663
processed: 1

raw_data:

{
    "added_by": 9,
    "added_when": 10,
    "company": 5,
    "expiry_date": 8,
    "for": 3,
    "for_id": 4,
    "id": 2,
    "newbook_region": 0,
    "policy_number": 7,
    "policy_type": 6,
    "tenant": 1
}

schema_attributes:

{
    "added_by": "string",
    "added_when": "string",
    "company": "string",
    "expiry_date": "string",
    "for": "string",
    "for_id": "string",
    "id": "string",
    "newbook_region": "string",
    "policy_number": "string",
    "policy_type": "string",
    "tenant": "string"
}

schema_name_id: 2671
schema_scan_id: 6667880
updated: 2024-11-05 02:16:19

Version 4741914

created: 2024-11-05 01:56:07

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `for`  string , `for_id`  string , `company`  string , `policy_type`  string , `policy_number`  string , `expiry_date`  string , `added_by`  string , `added_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730783667'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbookinsurance_policies"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730783667
id: 4741914
processed: 1

raw_data:

{
    "added_by": 9,
    "added_when": 10,
    "company": 5,
    "expiry_date": 8,
    "for": 3,
    "for_id": 4,
    "id": 2,
    "newbook_region": 0,
    "policy_number": 7,
    "policy_type": 6,
    "tenant": 1
}

schema_attributes:

{
    "added_by": "string",
    "added_when": "string",
    "company": "string",
    "expiry_date": "string",
    "for": "string",
    "for_id": "string",
    "id": "string",
    "newbook_region": "string",
    "policy_number": "string",
    "policy_type": "string",
    "tenant": "string"
}

schema_name_id: 2671
schema_scan_id: 6667131
updated: 2024-11-05 01:56:18

Version 4741138

created: 2024-11-05 01:35:47

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `for`  string , `for_id`  string , `company`  string , `policy_type`  string , `policy_number`  string , `expiry_date`  string , `added_by`  string , `added_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730783527'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbookinsurance_policies"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730783527
id: 4741138
processed: 1

raw_data:

{
    "added_by": 9,
    "added_when": 10,
    "company": 5,
    "expiry_date": 8,
    "for": 3,
    "for_id": 4,
    "id": 2,
    "newbook_region": 0,
    "policy_number": 7,
    "policy_type": 6,
    "tenant": 1
}

schema_attributes:

{
    "added_by": "string",
    "added_when": "string",
    "company": "string",
    "expiry_date": "string",
    "for": "string",
    "for_id": "string",
    "id": "string",
    "newbook_region": "string",
    "policy_number": "string",
    "policy_type": "string",
    "tenant": "string"
}

schema_name_id: 2671
schema_scan_id: 6666355
updated: 2024-11-05 01:35:58

Version 4740441

created: 2024-11-05 01:17:24

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `for`  string , `for_id`  string , `company`  string , `policy_type`  string , `policy_number`  string , `expiry_date`  string , `added_by`  string , `added_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730783309'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbookinsurance_policies"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730783309
id: 4740441
processed: 1

raw_data:

{
    "added_by": 9,
    "added_when": 10,
    "company": 5,
    "expiry_date": 8,
    "for": 3,
    "for_id": 4,
    "id": 2,
    "newbook_region": 0,
    "policy_number": 7,
    "policy_type": 6,
    "tenant": 1
}

schema_attributes:

{
    "added_by": "string",
    "added_when": "string",
    "company": "string",
    "expiry_date": "string",
    "for": "string",
    "for_id": "string",
    "id": "string",
    "newbook_region": "string",
    "policy_number": "string",
    "policy_type": "string",
    "tenant": "string"
}

schema_name_id: 2671
schema_scan_id: 6665658
updated: 2024-11-05 01:17:35

Version 4739642

created: 2024-11-05 00:56:00

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `for`  string , `for_id`  string , `company`  string , `policy_type`  string , `policy_number`  string , `expiry_date`  string , `added_by`  string , `added_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730782910'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbookinsurance_policies"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730782910
id: 4739642
processed: 1

raw_data:

{
    "added_by": 9,
    "added_when": 10,
    "company": 5,
    "expiry_date": 8,
    "for": 3,
    "for_id": 4,
    "id": 2,
    "newbook_region": 0,
    "policy_number": 7,
    "policy_type": 6,
    "tenant": 1
}

schema_attributes:

{
    "added_by": "string",
    "added_when": "string",
    "company": "string",
    "expiry_date": "string",
    "for": "string",
    "for_id": "string",
    "id": "string",
    "newbook_region": "string",
    "policy_number": "string",
    "policy_type": "string",
    "tenant": "string"
}

schema_name_id: 2671
schema_scan_id: 6664859
updated: 2024-11-05 00:56:14

Version 4738726

created: 2024-11-05 00:31:58

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `for`  string , `for_id`  string , `company`  string , `policy_type`  string , `policy_number`  string , `expiry_date`  string , `added_by`  string , `added_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730782808'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbookinsurance_policies"
]

hive_path: s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730782808
id: 4738726
processed: 1

raw_data:

{
    "added_by": 9,
    "added_when": 10,
    "company": 5,
    "expiry_date": 8,
    "for": 3,
    "for_id": 4,
    "id": 2,
    "newbook_region": 0,
    "policy_number": 7,
    "policy_type": 6,
    "tenant": 1
}

schema_attributes:

{
    "added_by": "string",
    "added_when": "string",
    "company": "string",
    "expiry_date": "string",
    "for": "string",
    "for_id": "string",
    "id": "string",
    "newbook_region": "string",
    "policy_number": "string",
    "policy_type": "string",
    "tenant": "string"
}

schema_name_id: 2671
schema_scan_id: 6663939
updated: 2024-11-05 00:32:09

Schema Scans: 15

Last at 2024-11-05 23:17:08

Scan 6675089

duration: 0:00:11.687362

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6675089

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-05/1730866516",
        "key": "raw/newbookdev_newbookinsurance_policies/2024-11-05/1730866516/insurance_policies.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbookinsurance_policies",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 4937701695160697929,
        "service_name": "drive-thru",
        "span_id": 6122103795668061041,
        "trace_id": 16162615378905373067
    }
}

running: False
schema_name_id: 2671
start_time: 2024-11-05 23:17:08
success: True
trace_id: 16162615378905373067

Scan 6674578

duration: 0:00:14.118155

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6674578

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-05/1730866443",
        "key": "raw/newbookdev_newbookinsurance_policies/2024-11-05/1730866443/insurance_policies.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbookinsurance_policies",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 11533136072457412154,
        "service_name": "drive-thru",
        "span_id": 12620755324564763652,
        "trace_id": 7687252639531851225
    }
}

running: False
schema_name_id: 2671
start_time: 2024-11-05 22:50:35
success: True
trace_id: 7687252639531851225

Scan 6673876

duration: 0:00:14.141734

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6673876

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-05/1730866319",
        "key": "raw/newbookdev_newbookinsurance_policies/2024-11-05/1730866319/insurance_policies.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbookinsurance_policies",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 3116937440720401694,
        "service_name": "drive-thru",
        "span_id": 8454670796014177132,
        "trace_id": 9311036289470226646
    }
}

running: False
schema_name_id: 2671
start_time: 2024-11-05 22:27:29
success: True
trace_id: 9311036289470226646

Scan 6672109

duration: 0:00:11.625464

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6672109

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730785727",
        "key": "raw/newbookdev_newbookinsurance_policies/2024-11-04/1730785727/insurance_policies.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbookinsurance_policies",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 3706763895336749614,
        "service_name": "drive-thru",
        "span_id": 5326737757836018932,
        "trace_id": 6496919173039710302
    }
}

running: False
schema_name_id: 2671
start_time: 2024-11-05 04:29:50
success: True
trace_id: 6496919173039710302

Scan 6670705

duration: 0:00:14.222268

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6670705

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730785128",
        "key": "raw/newbookdev_newbookinsurance_policies/2024-11-04/1730785128/insurance_policies.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbookinsurance_policies",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 9946447978134916834,
        "service_name": "drive-thru",
        "span_id": 12940054110056376369,
        "trace_id": 5566373655008867978
    }
}

running: False
schema_name_id: 2671
start_time: 2024-11-05 03:30:46
success: True
trace_id: 5566373655008867978

Scan 6670683

duration: 0:00:11.466697

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6670683

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730784823",
        "key": "raw/newbookdev_newbookinsurance_policies/2024-11-04/1730784823/insurance_policies.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbookinsurance_policies",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 14145016830222209507,
        "service_name": "drive-thru",
        "span_id": 3530023959622013660,
        "trace_id": 16310058847890376081
    }
}

running: False
schema_name_id: 2671
start_time: 2024-11-05 03:30:08
success: True
trace_id: 16310058847890376081

Scan 6669840

duration: 0:00:11.548932

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6669840

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730784219",
        "key": "raw/newbookdev_newbookinsurance_policies/2024-11-04/1730784219/insurance_policies.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbookinsurance_policies",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 18343802886149825375,
        "service_name": "drive-thru",
        "span_id": 17344888190785438896,
        "trace_id": 9246556832345403776
    }
}

running: False
schema_name_id: 2671
start_time: 2024-11-05 03:08:00
success: True
trace_id: 9246556832345403776

Scan 6669232

duration: 0:00:11.423443

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6669232

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730784057",
        "key": "raw/newbookdev_newbookinsurance_policies/2024-11-04/1730784057/insurance_policies.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbookinsurance_policies",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 14095051981466073462,
        "service_name": "drive-thru",
        "span_id": 1455591300550728996,
        "trace_id": 18230603841900063626
    }
}

running: False
schema_name_id: 2671
start_time: 2024-11-05 02:51:49
success: True
trace_id: 18230603841900063626

Scan 6668595

duration: 0:00:11.506547

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6668595

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730783901",
        "key": "raw/newbookdev_newbookinsurance_policies/2024-11-04/1730783901/insurance_policies.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbookinsurance_policies",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 17888963070836945484,
        "service_name": "drive-thru",
        "span_id": 4484112665170552085,
        "trace_id": 727572165076709570
    }
}

running: False
schema_name_id: 2671
start_time: 2024-11-05 02:35:00
success: True
trace_id: 727572165076709570

Scan 6667880

duration: 0:00:11.453246

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6667880

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730783771",
        "key": "raw/newbookdev_newbookinsurance_policies/2024-11-04/1730783771/insurance_policies.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbookinsurance_policies",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 16863781154732823302,
        "service_name": "drive-thru",
        "span_id": 13508941774439946044,
        "trace_id": 14749559196710062562
    }
}

running: False
schema_name_id: 2671
start_time: 2024-11-05 02:16:07
success: True
trace_id: 14749559196710062562

Scan 6667131

duration: 0:00:11.607502

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6667131

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730783667",
        "key": "raw/newbookdev_newbookinsurance_policies/2024-11-04/1730783667/insurance_policies.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbookinsurance_policies",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 5351480247455963988,
        "service_name": "drive-thru",
        "span_id": 18315568335564877569,
        "trace_id": 6388403637972230917
    }
}

running: False
schema_name_id: 2671
start_time: 2024-11-05 01:56:06
success: True
trace_id: 6388403637972230917

Scan 6666355

duration: 0:00:11.527195

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6666355

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730783527",
        "key": "raw/newbookdev_newbookinsurance_policies/2024-11-04/1730783527/insurance_policies.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbookinsurance_policies",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 2582451231889803243,
        "service_name": "drive-thru",
        "span_id": 1256171422127001551,
        "trace_id": 17602594033785572781
    }
}

running: False
schema_name_id: 2671
start_time: 2024-11-05 01:35:46
success: True
trace_id: 17602594033785572781

Scan 6665658

duration: 0:00:11.488605

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6665658

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730783309",
        "key": "raw/newbookdev_newbookinsurance_policies/2024-11-04/1730783309/insurance_policies.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbookinsurance_policies",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 16836496023521273539,
        "service_name": "drive-thru",
        "span_id": 16631612189285167899,
        "trace_id": 7630072200739013993
    }
}

running: False
schema_name_id: 2671
start_time: 2024-11-05 01:17:24
success: True
trace_id: 7630072200739013993

Scan 6664859

duration: 0:00:13.945256

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6664859

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730782910",
        "key": "raw/newbookdev_newbookinsurance_policies/2024-11-04/1730782910/insurance_policies.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbookinsurance_policies",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 14532064594348267509,
        "service_name": "drive-thru",
        "span_id": 2913492352052846984,
        "trace_id": 17075394680283587743
    }
}

running: False
schema_name_id: 2671
start_time: 2024-11-05 00:56:00
success: True
trace_id: 17075394680283587743

Scan 6663939

duration: 0:00:11.459837

exit_message:

{
    "ddl_changed": true,
    "partitions_applied": false,
    "success": true
}

id: 6663939

payload:

{
    "datatype_dict": null,
    "file_format": "csv",
    "file_properties": {
        "data_has_header_row": true
    },
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-drivethrucsv-drivethru-dev",
        "hive_path": "s3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-04/1730782808",
        "key": "raw/newbookdev_newbookinsurance_policies/2024-11-04/1730782808/insurance_policies.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_drivethru.newbookdev_newbookinsurance_policies",
        "type": "drivethrucsv"
    },
    "trace_context": {
        "parent_id": 3458869835165904213,
        "service_name": "drive-thru",
        "span_id": 14468523765474864543,
        "trace_id": 7433666883185071834
    }
}

running: False
schema_name_id: 2671
start_time: 2024-11-05 00:31:57
success: True
trace_id: 7433666883185071834

Partitions:

count: 0
ddl: None
name: None
state:
```
[]
```
type: None

Schema:

attributes:

{
    "added_by": "string",
    "added_when": "string",
    "company": "string",
    "expiry_date": "string",
    "for": "string",
    "for_id": "string",
    "id": "string",
    "newbook_region": "string",
    "policy_number": "string",
    "policy_type": "string",
    "tenant": "string"
}

created: 2024-11-05 23:17:09

ddl:

{
    "create_database": "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
    "create_table": "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `for`  string , `for_id`  string , `company`  string , `policy_type`  string , `policy_number`  string , `expiry_date`  string , `added_by`  string , `added_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-05/1730866516'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
    "drop_table": "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`",
    "repair_table": "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbookinsurance_policies"
}

Server:

hostname: schemamanager-5447774fc6-4w58v
name: schema_manager
process_start: 2026-04-02 21:43:44
sf_env: dev
version: 2427328685

Raw:

{
    "name": {
        "created": "2024-10-08 17:55:55",
        "disabled": false,
        "format": "raw",
        "id": 2671,
        "is_partitioned": false,
        "is_schema_datatyped": true,
        "name": {
            "database": "sfdl_dev_drivethru",
            "table": "newbookdev_newbookinsurance_policies"
        },
        "type": "drivethrucsv"
    },
    "partition": {
        "count": 0,
        "ddl": null,
        "name": null,
        "state": [],
        "type": null
    },
    "schema": {
        "attributes": {
            "added_by": "string",
            "added_when": "string",
            "company": "string",
            "expiry_date": "string",
            "for": "string",
            "for_id": "string",
            "id": "string",
            "newbook_region": "string",
            "policy_number": "string",
            "policy_type": "string",
            "tenant": "string"
        },
        "created": "2024-11-05 23:17:09",
        "ddl": {
            "create_database": "CREATE DATABASE IF NOT EXISTS `sfdl_dev_drivethru`",
            "create_table": "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`\n(\n `newbook_region`  string , `tenant`  string , `id`  string , `for`  string , `for_id`  string , `company`  string , `policy_type`  string , `policy_number`  string , `expiry_date`  string , `added_by`  string , `added_when`  string \n)\nROW FORMAT SERDE\n'org.apache.hadoop.hive.serde2.OpenCSVSerde'\n WITH SERDEPROPERTIES ( 'separatorChar' = ',', 'quoteChar' = '\"', 'escapeChar' = '\\\\')\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-drivethrucsv-drivethru-dev/raw/newbookdev_newbookinsurance_policies/2024-11-05/1730866516'\nTBLPROPERTIES ('skip.header.line.count' = '1')",
            "drop_table": "DROP TABLE IF EXISTS `sfdl_dev_drivethru.newbookdev_newbookinsurance_policies`",
            "repair_table": "MSCK REPAIR TABLE sfdl_dev_drivethru.newbookdev_newbookinsurance_policies"
        }
    },
    "server": {
        "hostname": "schemamanager-5447774fc6-4w58v",
        "name": "schema_manager",
        "process_start": "2026-04-02 21:43:44",
        "sf_env": "dev",
        "version": "2427328685"
    }
}