Schemas dev - Schema Manager

Schema Name: sfdl_dev_rds.sparefoot_billable_instances

created: 2022-03-07 18:37:48
disabled: False
format: raw
id: 1737
is_partitioned: False
is_schema_datatyped: True

name:

{
    "database": "sfdl_dev_rds",
    "table": "sparefoot_billable_instances"
}

type: rds

Schema Versions: 3

Version 4689076

created: 2022-03-16 17:20:03

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_rds`",
    "DROP TABLE IF EXISTS `sfdl_dev_rds.sparefoot_billable_instances`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_rds.sparefoot_billable_instances`\n(\n`unit_price`decimal(6,2),`invoice_id`int,`modified_dtm`timestamp,`statement_batch_id`int,`confirmation_code`varchar(32),`account_id`int,`sparefoot_charge`decimal(6,2),`residual_percent`decimal(3,2),`amount_collected`decimal(6,2),`listing_avail_id`int,`reason`varchar(32),`billable_instance_id`int,`statement_id`int,`item`varchar(128)\n)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-rds-sparefoot-dev/raw/billable_instances/2022-03-16/1647468845'\n",
    "MSCK REPAIR TABLE sfdl_dev_rds.sparefoot_billable_instances"
]

hive_path: s3://sfdl-rds-sparefoot-dev/raw/billable_instances/2022-03-16/1647468845
id: 4689076
processed: 1

raw_data:

{
    "account_id": 8046,
    "amount_collected": 0.0,
    "billable_instance_id": 7016393,
    "confirmation_code": "3TV7MKKPQ",
    "invoice_id": null,
    "item": "cpa_booking",
    "listing_avail_id": 208429,
    "modified_dtm": "2021-05-01 13:45:28",
    "reason": "canceled",
    "residual_percent": null,
    "sparefoot_charge": 0.0,
    "statement_batch_id": 138,
    "statement_id": 278792,
    "unit_price": 90.0
}

schema_attributes:

{
    "account_id": "int",
    "amount_collected": "string",
    "billable_instance_id": "int",
    "confirmation_code": "string",
    "invoice_id": "int",
    "item": "string",
    "listing_avail_id": "int",
    "modified_dtm": "timestamp",
    "reason": "string",
    "residual_percent": "string",
    "sparefoot_charge": "string",
    "statement_batch_id": "int",
    "statement_id": "int",
    "unit_price": "string"
}

schema_name_id: 1737
schema_scan_id: 5904849
updated: 2022-03-16 17:20:14

Version 4688950

created: 2022-03-15 20:34:07

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_rds`",
    "DROP TABLE IF EXISTS `sfdl_dev_rds.sparefoot_billable_instances`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_rds.sparefoot_billable_instances`\n(\n`unit_price`decimal(6,2),`invoice_id`int,`modified_dtm`timestamp,`statement_batch_id`int,`confirmation_code`varchar(32),`account_id`int,`sparefoot_charge`decimal(6,2),`residual_percent`decimal(3,2),`statement_id`int,`amount_collected`decimal(6,2),`listing_avail_id`int,`reason`varchar(32),`billable_instance_id`int,`item`varchar(128)\n)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-rds-sparefoot-dev/raw/billable_instances/2022-03-15/1647394419'\n",
    "MSCK REPAIR TABLE sfdl_dev_rds.sparefoot_billable_instances"
]

hive_path: s3://sfdl-rds-sparefoot-dev/raw/billable_instances/2022-03-15/1647394419
id: 4688950
processed: 1

raw_data:

{
    "account_id": 2413,
    "amount_collected": 134.0,
    "billable_instance_id": 437525,
    "confirmation_code": "3D5NX46F",
    "invoice_id": 66556,
    "item": "residual_booking",
    "listing_avail_id": 105765,
    "modified_dtm": "2014-03-11 18:24:50",
    "reason": null,
    "residual_percent": 0.25,
    "sparefoot_charge": 33.5,
    "statement_batch_id": 51,
    "statement_id": 33135,
    "unit_price": 134.0
}

schema_attributes:

{
    "account_id": "int",
    "amount_collected": "string",
    "billable_instance_id": "int",
    "confirmation_code": "string",
    "invoice_id": "int",
    "item": "string",
    "listing_avail_id": "int",
    "modified_dtm": "timestamp",
    "reason": "string",
    "residual_percent": "string",
    "sparefoot_charge": "string",
    "statement_batch_id": "int",
    "statement_id": "int",
    "unit_price": "string"
}

schema_name_id: 1737
schema_scan_id: 5904331
updated: 2022-03-15 20:34:19

Version 4688762

created: 2022-03-07 18:43:51

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_rds`",
    "DROP TABLE IF EXISTS `sfdl_dev_rds.sparefoot_billable_instances`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_rds.sparefoot_billable_instances`\n(\n`billable_instance_id`int,`statement_batch_id`int,`account_id`int,`listing_avail_id`int,`item`varchar(128),`statement_id`int,`confirmation_code`varchar(32),`unit_price`decimal(6,2),`reason`varchar(32),`amount_collected`decimal(6,2),`residual_percent`decimal(3,2),`sparefoot_charge`decimal(6,2),`invoice_id`int,`modified_dtm`timestamp\n)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-rds-sparefoot-dev/raw/billable_instances/2022-03-07/1646699851'\n",
    "MSCK REPAIR TABLE sfdl_dev_rds.sparefoot_billable_instances"
]

hive_path: s3://sfdl-rds-sparefoot-dev/raw/billable_instances/2022-03-07/1646699851
id: 4688762
processed: 1

raw_data:

{
    "account_id": 8039,
    "amount_collected": null,
    "billable_instance_id": 7016379,
    "confirmation_code": "YKKQJPVCE",
    "invoice_id": 606224,
    "item": "cpa_booking",
    "listing_avail_id": 208319,
    "modified_dtm": "2021-05-11 02:32:58",
    "reason": null,
    "residual_percent": null,
    "sparefoot_charge": 120.15,
    "statement_batch_id": 138,
    "statement_id": 278791,
    "unit_price": 89.0
}

schema_attributes:

{
    "account_id": "int",
    "amount_collected": "string",
    "billable_instance_id": "int",
    "confirmation_code": "string",
    "invoice_id": "int",
    "item": "string",
    "listing_avail_id": "int",
    "modified_dtm": "timestamp",
    "reason": "string",
    "residual_percent": "string",
    "sparefoot_charge": "string",
    "statement_batch_id": "int",
    "statement_id": "int",
    "unit_price": "string"
}

schema_name_id: 1737
schema_scan_id: 5902403
updated: 2022-03-07 18:44:02

Schema Scans: 3

Last at 2022-03-16 17:14:39

Scan 5904849

duration: 0:05:37.381979

exit_message:

{
    "ddl_changed": true,
    "ingested_partitions": 0,
    "partitions_applied": false,
    "success": true
}

id: 5904849

payload:

{
    "datatype_dict": {
        "account_id": "int",
        "amount_collected": "decimal(6,2)",
        "billable_instance_id": "int",
        "confirmation_code": "varchar(32)",
        "invoice_id": "int",
        "item": "varchar(128)",
        "listing_avail_id": "int",
        "modified_dtm": "timestamp",
        "reason": "varchar(32)",
        "residual_percent": "decimal(3,2)",
        "sparefoot_charge": "decimal(6,2)",
        "statement_batch_id": "int",
        "statement_id": "int",
        "unit_price": "decimal(6,2)"
    },
    "file_format": "json",
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-rds-sparefoot-dev",
        "hive_path": "s3://sfdl-rds-sparefoot-dev/raw/billable_instances/2022-03-16/1647468845",
        "key": "raw/billable_instances/2022-03-16/1647468845/billable_instances-8.json.gz"
    },
    "schema": {
        "name": "sfdl_dev_rds.sparefoot_billable_instances",
        "type": "rds"
    },
    "trace_context": {
        "parent_id": 17176709398705042433,
        "service_name": "sparefoot",
        "span_id": 16210871282281525374,
        "trace_id": 17176709398705042433
    }
}

running: False
schema_name_id: 1737
start_time: 2022-03-16 17:14:39
success: True
trace_id: 17176709398705042433

Scan 5904331

duration: 0:00:37.172603

exit_message:

{
    "ddl_changed": true,
    "ingested_partitions": 0,
    "partitions_applied": false,
    "success": true
}

id: 5904331

payload:

{
    "datatype_dict": {
        "account_id": "int",
        "amount_collected": "decimal(6,2)",
        "billable_instance_id": "int",
        "confirmation_code": "varchar(32)",
        "invoice_id": "int",
        "item": "varchar(128)",
        "listing_avail_id": "int",
        "modified_dtm": "timestamp",
        "reason": "varchar(32)",
        "residual_percent": "decimal(3,2)",
        "sparefoot_charge": "decimal(6,2)",
        "statement_batch_id": "int",
        "statement_id": "int",
        "unit_price": "decimal(6,2)"
    },
    "file_format": "json",
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-rds-sparefoot-dev",
        "hive_path": "s3://sfdl-rds-sparefoot-dev/raw/billable_instances/2022-03-15/1647394419",
        "key": "raw/billable_instances/2022-03-15/1647394419/data-8.json.gz"
    },
    "schema": {
        "name": "sfdl_dev_rds.sparefoot_billable_instances",
        "type": "rds"
    },
    "trace_context": {
        "parent_id": 16188274156457561556,
        "service_name": "sparefoot",
        "span_id": 15991716588510244680,
        "trace_id": 16188274156457561556
    }
}

running: False
schema_name_id: 1737
start_time: 2022-03-15 20:33:44
success: True
trace_id: 16188274156457561556

Scan 5902403

duration: 0:06:16.836571

exit_message:

{
    "ddl_changed": true,
    "ingested_partitions": 0,
    "partitions_applied": false,
    "success": true
}

id: 5902403

payload:

{
    "datatype_dict": {
        "account_id": "int",
        "amount_collected": "decimal(6,2)",
        "billable_instance_id": "int",
        "confirmation_code": "varchar(32)",
        "invoice_id": "int",
        "item": "varchar(128)",
        "listing_avail_id": "int",
        "modified_dtm": "timestamp",
        "reason": "varchar(32)",
        "residual_percent": "decimal(3,2)",
        "sparefoot_charge": "decimal(6,2)",
        "statement_batch_id": "int",
        "statement_id": "int",
        "unit_price": "decimal(6,2)"
    },
    "file_format": "json",
    "is_schema_datatyped": true,
    "s3": {
        "bucket": "sfdl-rds-sparefoot-dev",
        "hive_path": "s3://sfdl-rds-sparefoot-dev/raw/billable_instances/2022-03-07/1646699851",
        "key": "raw/billable_instances/2022-03-07/1646699851/data-8.csv.gz"
    },
    "schema": {
        "name": "sfdl_dev_rds.sparefoot_billable_instances",
        "type": "rds"
    },
    "trace_context": {
        "parent_id": 15025767279777263963,
        "service_name": "sparefoot",
        "span_id": 2576784018189882232,
        "trace_id": 15025767279777263963
    }
}

running: False
schema_name_id: 1737
start_time: 2022-03-07 18:37:48
success: True
trace_id: 15025767279777263963

Partitions:

count: 0
ddl: None
name: None
state:
```
[]
```
type: None

Schema:

attributes:

{
    "account_id": "int",
    "amount_collected": "string",
    "billable_instance_id": "int",
    "confirmation_code": "string",
    "invoice_id": "int",
    "item": "string",
    "listing_avail_id": "int",
    "modified_dtm": "timestamp",
    "reason": "string",
    "residual_percent": "string",
    "sparefoot_charge": "string",
    "statement_batch_id": "int",
    "statement_id": "int",
    "unit_price": "string"
}

created: 2022-03-16 17:20:03

ddl:

{
    "create_database": "CREATE DATABASE IF NOT EXISTS `sfdl_dev_rds`",
    "create_table": "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_rds.sparefoot_billable_instances`\n(\n`unit_price`decimal(6,2),`invoice_id`int,`modified_dtm`timestamp,`statement_batch_id`int,`confirmation_code`varchar(32),`account_id`int,`sparefoot_charge`decimal(6,2),`residual_percent`decimal(3,2),`amount_collected`decimal(6,2),`listing_avail_id`int,`reason`varchar(32),`billable_instance_id`int,`statement_id`int,`item`varchar(128)\n)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-rds-sparefoot-dev/raw/billable_instances/2022-03-16/1647468845'\n",
    "drop_table": "DROP TABLE IF EXISTS `sfdl_dev_rds.sparefoot_billable_instances`",
    "repair_table": "MSCK REPAIR TABLE sfdl_dev_rds.sparefoot_billable_instances"
}

Server:

hostname: schemamanager-5447774fc6-4w58v
name: schema_manager
process_start: 2026-04-02 21:43:44
sf_env: dev
version: 2427328685

Raw:

{
    "name": {
        "created": "2022-03-07 18:37:48",
        "disabled": false,
        "format": "raw",
        "id": 1737,
        "is_partitioned": false,
        "is_schema_datatyped": true,
        "name": {
            "database": "sfdl_dev_rds",
            "table": "sparefoot_billable_instances"
        },
        "type": "rds"
    },
    "partition": {
        "count": 0,
        "ddl": null,
        "name": null,
        "state": [],
        "type": null
    },
    "schema": {
        "attributes": {
            "account_id": "int",
            "amount_collected": "string",
            "billable_instance_id": "int",
            "confirmation_code": "string",
            "invoice_id": "int",
            "item": "string",
            "listing_avail_id": "int",
            "modified_dtm": "timestamp",
            "reason": "string",
            "residual_percent": "string",
            "sparefoot_charge": "string",
            "statement_batch_id": "int",
            "statement_id": "int",
            "unit_price": "string"
        },
        "created": "2022-03-16 17:20:03",
        "ddl": {
            "create_database": "CREATE DATABASE IF NOT EXISTS `sfdl_dev_rds`",
            "create_table": "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_rds.sparefoot_billable_instances`\n(\n`unit_price`decimal(6,2),`invoice_id`int,`modified_dtm`timestamp,`statement_batch_id`int,`confirmation_code`varchar(32),`account_id`int,`sparefoot_charge`decimal(6,2),`residual_percent`decimal(3,2),`amount_collected`decimal(6,2),`listing_avail_id`int,`reason`varchar(32),`billable_instance_id`int,`statement_id`int,`item`varchar(128)\n)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-rds-sparefoot-dev/raw/billable_instances/2022-03-16/1647468845'\n",
            "drop_table": "DROP TABLE IF EXISTS `sfdl_dev_rds.sparefoot_billable_instances`",
            "repair_table": "MSCK REPAIR TABLE sfdl_dev_rds.sparefoot_billable_instances"
        }
    },
    "server": {
        "hostname": "schemamanager-5447774fc6-4w58v",
        "name": "schema_manager",
        "process_start": "2026-04-02 21:43:44",
        "sf_env": "dev",
        "version": "2427328685"
    }
}

Schema Info