Schema Info

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_segment`",
    "DROP TABLE IF EXISTS `sfdl_dev_segment.ssdc_identify`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_segment.ssdc_identify`\n(\n`traits`struct<`_empty_`:STRING>,`type`STRING,`version`STRING,`channel`STRING,`timestamp`STRING,`_metadata`struct<`bundled`:array,`unbundled`:array>,`integrations`struct<`_empty_`:STRING>,`_origin`STRING,`userId`STRING,`sentAt`STRING,`messageId`STRING,`projectId`STRING,`originalTimestamp`STRING,`anonymousId`STRING,`context`struct<`userAgent`:STRING,`timezone`:STRING,`ip`:STRING,`library`:struct<`version`:STRING,`name`:STRING>,`locale`:STRING,`page`:struct<`path`:STRING,`url`:STRING,`referrer`:STRING,`title`:STRING,`search`:STRING>,`userAgentData`:struct<`mobile`:STRING,`platform`:STRING,`brands`:array>>,`campaign`:struct<`_3Dgoogle_ppc_26utm_source_3Dgoogle_26utm_medium_3Dpaidsearch_26utm_campaign_3Dpro_regional_east_storage_exact_26utm_content_3Dcontrol_26ploc_3D9016801_26iloc_3D_26mkwid_3DsGCAF1AwP_7Cpcrid_7C263851863259_7Cpkw_7Cstorage_7Cpmt_7Ce_7Cpdv_7Cm_7Cpgrid_7C9675654331_7Cptaid_7Ckwd_10003413_26pgrid_3D9675654331_26ptaid_3Dkwd_10003413_26muid_3D_m_uid_`:STRING,`source`:STRING,`_3Dgoogle_ppc_26utm_source_3Dgoogle_26utm_medium_3Dpaidsearch_26utm_campaign_3Dpro_regional_east_storage_exact_26utm_content_3Dcontrol_26ploc_3D9016801_26iloc_3D_26mkwid_3DsGCAF1AwP_7Cpcrid_7C263851863259_7Cpkw_7Cstorage_7Cpmt_7Ce_7Cpdv_7Cm_7Cpgrid_7C9675654331`:STRING,`content`:STRING,`medium`:STRING,`name`:STRING,`_3Dgoogle_ppc_26utm_source_3Dgoogle_26utm_medium_3Dpaidsearch_26utm_campaign_3Dpro_regional_east_storage_exact_26utm_content_3Dcontrol_26ploc_3D9011659_26iloc_3D_26mkwid_3DsGCAF1AwP_7Cpcrid_7C263851863259_7Cpkw_7Cstorage_7Cpmt_7Ce_7Cpdv_7Cm_7Cpgrid_7C9675654331_7Cptaid_7Ckwd_10003413_26pgrid_3D9675654331_26ptaid_3Dkwd_10003413_26muid_3D_7Bm_uid_7D`:STRING>>,`receivedAt`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n WITH SERDEPROPERTIES ( 'mapping.ga_client_id' = 'GA Client ID' , 'mapping.GA_Client_ID' = 'GA Client ID' )\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-segment-ssdc-dev/separated_dt/identify'\n",
    "MSCK REPAIR TABLE sfdl_dev_segment.ssdc_identify"
]

{
    "_metadata": {
        "bundled": [
            "Google Analytics",
            "Google Tag Manager",
            "Segment.io"
        ],
        "bundledIds": [
            "IAdHmAh2FE",
            "5ab94e33d7cab31211fc1606"
        ],
        "unbundled": []
    },
    "_origin": "2023-11-17T16:51:17Z|s3://sfdl-segment-ssdc-dev/segment-logs/VCAARGbeZs/1700179200000/1700239618320.events-archiver-480-29..d5192bd4-ad7d-4d49-a098-28ce1886ca62.gz|analytics-segment-separator-dev:$LATEST",
    "anonymousId": "79ece842-4eaf-4cf2-a28d-ba790f0ed592",
    "channel": "client",
    "context": {
        "campaign": {},
        "ip": "187.142.13.115",
        "library": {
            "name": "analytics.js",
            "version": "next-1.61.0"
        },
        "locale": "en-US",
        "page": {
            "path": "/self-storage/texas/austin/",
            "referrer": "",
            "search": "?forceExperiments=sf_12164_ssdc_clp_redesign:variation",
            "title": "20 Cheap Storage Units in Austin, TX (from $21)",
            "url": "https://www.selfstorage.extrameter.com/self-storage/texas/austin/?forceExperiments=sf_12164_ssdc_clp_redesign:variation"
        },
        "timezone": "America/Mexico_City",
        "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
        "userAgentData": {
            "brands": [
                {
                    "brand": "Google Chrome",
                    "version": "119"
                },
                {
                    "brand": "Chromium",
                    "version": "119"
                },
                {
                    "brand": "Not?A_Brand",
                    "version": "24"
                }
            ],
            "mobile": false,
            "platform": "macOS"
        }
    },
    "integrations": {},
    "messageId": "ajs-next-8b4c7b82a2eda5e755cda096708bbad2",
    "originalTimestamp": "2023-11-17T16:18:46.106Z",
    "projectId": "VCAARGbeZs",
    "receivedAt": "2023-11-17T16:18:46.904Z",
    "sentAt": "2023-11-17T16:18:46.138Z",
    "timestamp": "2023-11-17T16:18:46.872Z",
    "traits": {},
    "type": "identify",
    "userId": "error",
    "version": 2
}

{
    "_origin": "string",
    "anonymousId": "string",
    "channel": "string",
    "context.campaign.content": "string",
    "context.campaign.medium": "string",
    "context.campaign.name": "string",
    "context.campaign.source": "string",
    "context.ip": "string",
    "context.library.name": "string",
    "context.library.version": "string",
    "context.locale": "string",
    "context.page.path": "string",
    "context.page.referrer": "string",
    "context.page.search": "string",
    "context.page.title": "string",
    "context.page.url": "string",
    "context.timezone": "string",
    "context.userAgent": "string",
    "context.userAgentData.brands": [
        {
            "brand": "string",
            "version": "string"
        }
    ],
    "context.userAgentData.mobile": "string",
    "context.userAgentData.platform": "string",
    "messageId": "string",
    "originalTimestamp": "string",
    "projectId": "string",
    "receivedAt": "string",
    "sentAt": "string",
    "timestamp": "string",
    "type": "string",
    "userId": "string",
    "version": "string"
}

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_segment`",
    "DROP TABLE IF EXISTS `sfdl_dev_segment.ssdc_identify`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_segment.ssdc_identify`\n(\n`sentAt`STRING,`version`STRING,`channel`STRING,`integrations`struct<`_empty_`:STRING>,`timestamp`STRING,`_metadata`struct<`bundled`:array,`unbundled`:array>,`type`STRING,`_origin`STRING,`messageId`STRING,`userId`STRING,`anonymousId`STRING,`context`struct<`locale`:STRING,`campaign`:struct<`_3Dgoogle_ppc_26utm_source_3Dgoogle_26utm_medium_3Dpaidsearch_26utm_campaign_3Dpro_regional_east_storage_exact_26utm_content_3Dcontrol_26ploc_3D9016801_26iloc_3D_26mkwid_3DsGCAF1AwP_7Cpcrid_7C263851863259_7Cpkw_7Cstorage_7Cpmt_7Ce_7Cpdv_7Cm_7Cpgrid_7C9675654331_7Cptaid_7Ckwd_10003413_26pgrid_3D9675654331_26ptaid_3Dkwd_10003413_26muid_3D_m_uid_`:STRING,`source`:STRING,`_3Dgoogle_ppc_26utm_source_3Dgoogle_26utm_medium_3Dpaidsearch_26utm_campaign_3Dpro_regional_east_storage_exact_26utm_content_3Dcontrol_26ploc_3D9016801_26iloc_3D_26mkwid_3DsGCAF1AwP_7Cpcrid_7C263851863259_7Cpkw_7Cstorage_7Cpmt_7Ce_7Cpdv_7Cm_7Cpgrid_7C9675654331`:STRING,`content`:STRING,`medium`:STRING,`name`:STRING,`_3Dgoogle_ppc_26utm_source_3Dgoogle_26utm_medium_3Dpaidsearch_26utm_campaign_3Dpro_regional_east_storage_exact_26utm_content_3Dcontrol_26ploc_3D9011659_26iloc_3D_26mkwid_3DsGCAF1AwP_7Cpcrid_7C263851863259_7Cpkw_7Cstorage_7Cpmt_7Ce_7Cpdv_7Cm_7Cpgrid_7C9675654331_7Cptaid_7Ckwd_10003413_26pgrid_3D9675654331_26ptaid_3Dkwd_10003413_26muid_3D_7Bm_uid_7D`:STRING>,`library`:struct<`version`:STRING,`name`:STRING>,`userAgent`:STRING,`page`:struct<`url`:STRING,`path`:STRING,`title`:STRING,`referrer`:STRING,`search`:STRING>,`ip`:STRING,`userAgentData`:struct<`brands`:array>,`mobile`:STRING,`platform`:STRING>>,`traits`struct<`_empty_`:STRING>,`originalTimestamp`STRING,`receivedAt`STRING,`projectId`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n WITH SERDEPROPERTIES ( 'mapping.ga_client_id' = 'GA Client ID' , 'mapping.GA_Client_ID' = 'GA Client ID' )\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-segment-ssdc-dev/separated_dt/identify'\n",
    "MSCK REPAIR TABLE sfdl_dev_segment.ssdc_identify"
]

{
    "_metadata": {
        "bundled": [
            "Google Analytics",
            "Google Tag Manager",
            "Segment.io"
        ],
        "bundledIds": [
            "IAdHmAh2FE",
            "5ab94e33d7cab31211fc1606"
        ],
        "unbundled": []
    },
    "_origin": "2023-06-09T14:13:12Z|s3://sfdl-segment-ssdc-dev/segment-logs/VCAARGbeZs/1686268800000/1686319894293.events-archiver-480-48.8ebb200.abffa4ac-5e46-4031-876a-9482f56f44e5.gz|analytics-segment-separator-dev:$LATEST",
    "anonymousId": "c899778a-e319-423a-a4a8-55e353901df5",
    "channel": "client",
    "context": {
        "ip": "72.133.65.251",
        "library": {
            "name": "analytics.js",
            "version": "next-1.53.0"
        },
        "locale": "en-US",
        "page": {
            "path": "/self-storage/texas/austin/",
            "referrer": "",
            "search": "",
            "title": "20 Cheap Storage Units in Austin, TX (from $21)",
            "url": "https://www.selfstorage.moreyard.com/self-storage/texas/austin/"
        },
        "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
        "userAgentData": {
            "brands": [
                {
                    "brand": "Not.A/Brand",
                    "version": "8"
                },
                {
                    "brand": "Chromium",
                    "version": "114"
                },
                {
                    "brand": "Google Chrome",
                    "version": "114"
                }
            ],
            "mobile": false,
            "platform": "macOS"
        }
    },
    "integrations": {},
    "messageId": "ajs-next-c5150682bf49dddcca1a18c00def6b1e",
    "originalTimestamp": "2023-06-09T14:05:11.435Z",
    "projectId": "VCAARGbeZs",
    "receivedAt": "2023-06-09T14:05:12.008Z",
    "sentAt": "2023-06-09T14:05:11.457Z",
    "timestamp": "2023-06-09T14:05:11.986Z",
    "traits": {},
    "type": "identify",
    "userId": "21d9b94820818f07aee6c0836a817e1b",
    "version": 2
}

{
    "_origin": "string",
    "anonymousId": "string",
    "channel": "string",
    "context.campaign.content": "string",
    "context.campaign.medium": "string",
    "context.campaign.name": "string",
    "context.campaign.source": "string",
    "context.ip": "string",
    "context.library.name": "string",
    "context.library.version": "string",
    "context.locale": "string",
    "context.page.path": "string",
    "context.page.referrer": "string",
    "context.page.search": "string",
    "context.page.title": "string",
    "context.page.url": "string",
    "context.userAgent": "string",
    "context.userAgentData.brands": [
        {
            "brand": "string",
            "version": "string"
        }
    ],
    "context.userAgentData.mobile": "string",
    "context.userAgentData.platform": "string",
    "messageId": "string",
    "originalTimestamp": "string",
    "projectId": "string",
    "receivedAt": "string",
    "sentAt": "string",
    "timestamp": "string",
    "type": "string",
    "userId": "string",
    "version": "string"
}

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_dev_segment`",
    "DROP TABLE IF EXISTS `sfdl_dev_segment.ssdc_identify`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_dev_segment.ssdc_identify`\n(\n`receivedAt`STRING,`userId`STRING,`type`STRING,`version`STRING,`projectId`STRING,`_origin`STRING,`timestamp`STRING,`traits`struct<`_empty_`:STRING>,`channel`STRING,`messageId`STRING,`integrations`struct<`_empty_`:STRING>,`originalTimestamp`STRING,`anonymousId`STRING,`sentAt`STRING,`context`struct<`ip`:STRING,`campaign`:struct<`_3Dgoogle_ppc_26utm_source_3Dgoogle_26utm_medium_3Dpaidsearch_26utm_campaign_3Dpro_regional_east_storage_exact_26utm_content_3Dcontrol_26ploc_3D9016801_26iloc_3D_26mkwid_3DsGCAF1AwP_7Cpcrid_7C263851863259_7Cpkw_7Cstorage_7Cpmt_7Ce_7Cpdv_7Cm_7Cpgrid_7C9675654331_7Cptaid_7Ckwd_10003413_26pgrid_3D9675654331_26ptaid_3Dkwd_10003413_26muid_3D_m_uid_`:STRING,`source`:STRING,`_3Dgoogle_ppc_26utm_source_3Dgoogle_26utm_medium_3Dpaidsearch_26utm_campaign_3Dpro_regional_east_storage_exact_26utm_content_3Dcontrol_26ploc_3D9016801_26iloc_3D_26mkwid_3DsGCAF1AwP_7Cpcrid_7C263851863259_7Cpkw_7Cstorage_7Cpmt_7Ce_7Cpdv_7Cm_7Cpgrid_7C9675654331`:STRING,`content`:STRING,`medium`:STRING,`name`:STRING,`_3Dgoogle_ppc_26utm_source_3Dgoogle_26utm_medium_3Dpaidsearch_26utm_campaign_3Dpro_regional_east_storage_exact_26utm_content_3Dcontrol_26ploc_3D9011659_26iloc_3D_26mkwid_3DsGCAF1AwP_7Cpcrid_7C263851863259_7Cpkw_7Cstorage_7Cpmt_7Ce_7Cpdv_7Cm_7Cpgrid_7C9675654331_7Cptaid_7Ckwd_10003413_26pgrid_3D9675654331_26ptaid_3Dkwd_10003413_26muid_3D_7Bm_uid_7D`:STRING>,`library`:struct<`name`:STRING,`version`:STRING>,`page`:struct<`url`:STRING,`search`:STRING,`path`:STRING,`referrer`:STRING,`title`:STRING>,`userAgent`:STRING,`locale`:STRING>,`_metadata`struct<`bundled`:array,`unbundled`:array>\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n WITH SERDEPROPERTIES ( 'mapping.ga_client_id' = 'GA Client ID' , 'mapping.GA_Client_ID' = 'GA Client ID' )\nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-segment-ssdc-dev/separated_dt/identify'\n",
    "MSCK REPAIR TABLE sfdl_dev_segment.ssdc_identify"
]

{
    "_metadata": {
        "bundled": [
            "Google Analytics",
            "Google Tag Manager",
            "Segment.io"
        ],
        "bundledIds": [
            "IAdHmAh2FE",
            "5ab94e33d7cab31211fc1606"
        ],
        "unbundled": []
    },
    "_origin": "2021-07-22T22:46:39Z|s3://sfdl-segment-ssdc-dev/segment-logs/VCAARGbeZs/1626912000000/1626993549210.efda54410a74.8624813.f897e1c0-c9b3-4615-ba8e-b46f3b6835e3.gz|analytics-segment-separator-dev:$LATEST",
    "anonymousId": "967b3728-c8e8-42d3-954e-8dfe6cafda7b",
    "channel": "client",
    "context": {
        "campaign": {
            "%3Dgoogle-ppc%26utm_source%3Dgoogle%26utm_medium%3Dpaidsearch%26utm_campaign%3Dpro_regional_east_storage_exact%26utm_content%3Dcontrol%26ploc%3D9011659%26iloc%3D%26mkwid%3DsGCAF1AwP%7Cpcrid%7C263851863259%7Cpkw%7Cstorage%7Cpmt%7Ce%7Cpdv%7Cm%7Cpgrid%7C9675654331%7Cptaid%7Ckwd-10003413%26pgrid%3D9675654331%26ptaid%3Dkwd-10003413%26muid%3D%7Bm_uid%7D": "",
            "content": "control",
            "medium": "paidsearch",
            "name": "pro_regional_east_storage_exact",
            "source": "google"
        },
        "ip": "34.239.223.223",
        "library": {
            "name": "analytics.js",
            "version": "next-1.25.3"
        },
        "locale": "en-US",
        "page": {
            "path": "/storage-units-near-me",
            "referrer": "",
            "search": "?tsid%3Dgoogle-ppc%26utm_source%3Dgoogle%26utm_medium%3Dpaidsearch%26utm_campaign%3Dpro_regional_east_storage_exact%26utm_content%3Dcontrol%26ploc%3D9011659%26iloc%3D%26mkwid%3DsGCAF1AwP%7Cpcrid%7C263851863259%7Cpkw%7Cstorage%7Cpmt%7Ce%7Cpdv%7Cm%7Cpgrid%7C9675654331%7Cptaid%7Ckwd-10003413%26pgrid%3D9675654331%26ptaid%3Dkwd-10003413%26muid%3D%7Bm_uid%7D&gclid=EAIaIQobChMIwPu5t4qs3AIVAQAAAB0BAAAAEAAYACAAEgJVzfD_BwE",
            "title": "Find Cheap Self-Storage Units Near You - SelfStorage.com",
            "url": "http://localhost:9116/storage-units-near-me?tsid%3Dgoogle-ppc%26utm_source%3Dgoogle%26utm_medium%3Dpaidsearch%26utm_campaign%3Dpro_regional_east_storage_exact%26utm_content%3Dcontrol%26ploc%3D9011659%26iloc%3D%26mkwid%3DsGCAF1AwP%7Cpcrid%7C263851863259%7Cpkw%7Cstorage%7Cpmt%7Ce%7Cpdv%7Cm%7Cpgrid%7C9675654331%7Cptaid%7Ckwd-10003413%26pgrid%3D9675654331%26ptaid%3Dkwd-10003413%26muid%3D%7Bm_uid%7D&gclid=EAIaIQobChMIwPu5t4qs3AIVAQAAAB0BAAAAEAAYACAAEgJVzfD_BwE"
        },
        "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36"
    },
    "integrations": {},
    "messageId": "ajs-next-fdb44d6d4ade7930a1bf937dbafc12c9",
    "originalTimestamp": "2021-07-22T21:55:58.034Z",
    "projectId": "VCAARGbeZs",
    "receivedAt": "2021-07-22T21:55:58.561Z",
    "sentAt": "2021-07-22T21:55:58.034Z",
    "timestamp": "2021-07-22T21:55:58.561Z",
    "traits": {},
    "type": "identify",
    "userId": "error",
    "version": 2
}

{
    "_origin": "string",
    "anonymousId": "string",
    "channel": "string",
    "context.campaign._3Dgoogle_ppc_26utm_source_3Dgoogle_26utm_medium_3Dpaidsearch_26utm_campaign_3Dpro_regional_east_storage_exact_26utm_content_3Dcontrol_26ploc_3D9011659_26iloc_3D_26mkwid_3DsGCAF1AwP_7Cpcrid_7C263851863259_7Cpkw_7Cstorage_7Cpmt_7Ce_7Cpdv_7Cm_7Cpgrid_7C9675654331_7Cptaid_7Ckwd_10003413_26pgrid_3D9675654331_26ptaid_3Dkwd_10003413_26muid_3D_7Bm_uid_7D": "string",
    "context.campaign._3Dgoogle_ppc_26utm_source_3Dgoogle_26utm_medium_3Dpaidsearch_26utm_campaign_3Dpro_regional_east_storage_exact_26utm_content_3Dcontrol_26ploc_3D9016801_26iloc_3D_26mkwid_3DsGCAF1AwP_7Cpcrid_7C263851863259_7Cpkw_7Cstorage_7Cpmt_7Ce_7Cpdv_7Cm_7Cpgrid_7C9675654331": "string",
    "context.campaign._3Dgoogle_ppc_26utm_source_3Dgoogle_26utm_medium_3Dpaidsearch_26utm_campaign_3Dpro_regional_east_storage_exact_26utm_content_3Dcontrol_26ploc_3D9016801_26iloc_3D_26mkwid_3DsGCAF1AwP_7Cpcrid_7C263851863259_7Cpkw_7Cstorage_7Cpmt_7Ce_7Cpdv_7Cm_7Cpgrid_7C9675654331_7Cptaid_7Ckwd_10003413_26pgrid_3D9675654331_26ptaid_3Dkwd_10003413_26muid_3D_m_uid_": "string",
    "context.campaign.content": "string",
    "context.campaign.medium": "string",
    "context.campaign.name": "string",
    "context.campaign.source": "string",
    "context.ip": "string",
    "context.library.name": "string",
    "context.library.version": "string",
    "context.locale": "string",
    "context.page.path": "string",
    "context.page.referrer": "string",
    "context.page.search": "string",
    "context.page.title": "string",
    "context.page.url": "string",
    "context.userAgent": "string",
    "messageId": "string",
    "originalTimestamp": "string",
    "projectId": "string",
    "receivedAt": "string",
    "sentAt": "string",
    "timestamp": "string",
    "type": "string",
    "userId": "string",
    "version": "string"
}

{
    "ddl_changed": true,
    "ingested_partitions": 1673,
    "partitions_applied": false,
    "success": true
}

{
    "datatype_dict": {
        "originalTimestamp": "timestamp_iso8601",
        "receivedAt": "timestamp_iso8601",
        "sentAt": "timestamp_iso8601",
        "timestamp": "timestamp_iso8601"
    },
    "file_format": "json",
    "is_schema_datatyped": false,
    "partition": {
        "location": "s3://sfdl-segment-ssdc-dev/separated_dt/identify/dt=2023-11-17/",
        "name": "dt",
        "type": "string",
        "value": "2023-11-17"
    },
    "s3": {
        "bucket": "sfdl-segment-ssdc-dev",
        "hive_path": "s3://sfdl-segment-ssdc-dev/separated_dt/identify",
        "key": "separated_dt/identify/dt=2023-11-17/1700239618320.events-archiver-480-29..d5192bd4-ad7d-4d49-a098-28ce1886ca62.json.gz"
    },
    "schema": {
        "name": "sfdl_dev_segment.ssdc_identify",
        "type": "segment"
    }
}

{
    "exception": "failed to insert partition: (psycopg2.errors.UniqueViolation) duplicate key value violates unique constraint \"partitions_schema_name_id_value_name_key\"\nDETAIL:  Key (schema_name_id, value, name)=(352, 2017-06-03, dt) already exists.\n\n[SQL: INSERT INTO partitions (schema_name_id, name, type, value, location, processed, created, data_first_arrived, data_last_arrived, parquet_first_performed, parquet_last_performed, parquet_in_process, parquet_job_id, flag_for_parquet) VALUES (%(schema_name_id)s, %(name)s, %(type)s, %(value)s, %(location)s, %(processed)s, %(created)s, %(data_first_arrived)s, %(data_last_arrived)s, %(parquet_first_performed)s, %(parquet_last_performed)s, %(parquet_in_process)s, %(parquet_job_id)s, %(flag_for_parquet)s) RETURNING partitions.id]\n[parameters: {'schema_name_id': 352, 'name': 'dt', 'type': 'string', 'value': '2017-06-03', 'location': 's3://sfdl-segment-ssdc-dev/separated_dt/identify/dt=2017-06-03/', 'processed': 0, 'created': '2023-06-09 09:22:53.533938', 'data_first_arrived': '2023-06-09 09:22:53.533938', 'data_last_arrived': '2023-06-09 09:22:53.533938', 'parquet_first_performed': None, 'parquet_last_performed': None, 'parquet_in_process': None, 'parquet_job_id': None, 'flag_for_parquet': True}]\n(Background on this error at: https://sqlalche.me/e/14/gkpj)\nTraceback (most recent call last):\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py\", line 1900, in _execute_context\n    self.dialect.do_execute(\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/default.py\", line 736, in do_execute\n    cursor.execute(statement, parameters)\npsycopg2.errors.UniqueViolation: duplicate key value violates unique constraint \"partitions_schema_name_id_value_name_key\"\nDETAIL:  Key (schema_name_id, value, name)=(352, 2017-06-03, dt) already exists.\n\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File \"/schema_manager/schema_manager/service/partition_bag.py\", line 114, in _insert\n    session.commit()\n  File \"\", line 2, in commit\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py\", line 1451, in commit\n    self._transaction.commit(_to_root=self.future)\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py\", line 829, in commit\n    self._prepare_impl()\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py\", line 808, in _prepare_impl\n    self.session.flush()\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py\", line 3429, in flush\n    self._flush(objects)\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py\", line 3568, in _flush\n    with util.safe_reraise():\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py\", line 70, in __exit__\n    compat.raise_(\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/util/compat.py\", line 208, in raise_\n    raise exception\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py\", line 3529, in _flush\n    flush_context.execute()\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/unitofwork.py\", line 456, in execute\n    rec.execute(self)\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/unitofwork.py\", line 630, in execute\n    util.preloaded.orm_persistence.save_obj(\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/persistence.py\", line 245, in save_obj\n    _emit_insert_statements(\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/persistence.py\", line 1238, in _emit_insert_statements\n    result = connection._execute_20(\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py\", line 1705, in _execute_20\n    return meth(self, args_10style, kwargs_10style, execution_options)\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/sql/elements.py\", line 333, in _execute_on_connection\n    return connection._execute_clauseelement(\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py\", line 1572, in _execute_clauseelement\n    ret = self._execute_context(\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py\", line 1943, in _execute_context\n    self._handle_dbapi_exception(\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py\", line 2124, in _handle_dbapi_exception\n    util.raise_(\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/util/compat.py\", line 208, in raise_\n    raise exception\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py\", line 1900, in _execute_context\n    self.dialect.do_execute(\n  File \"/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/default.py\", line 736, in do_execute\n    cursor.execute(statement, parameters)\nsqlalchemy.exc.IntegrityError: (psycopg2.errors.UniqueViolation) duplicate key value violates unique constraint \"partitions_schema_name_id_value_name_key\"\nDETAIL:  Key (schema_name_id, value, name)=(352, 2017-06-03, dt) already exists.\n\n[SQL: INSERT INTO partitions (schema_name_id, name, type, value, location, processed, created, data_first_arrived, data_last_arrived, parquet_first_performed, parquet_last_performed, parquet_in_process, parquet_job_id, flag_for_parquet) VALUES (%(schema_name_id)s, %(name)s, %(type)s, %(value)s, %(location)s, %(processed)s, %(created)s, %(data_first_arrived)s, %(data_last_arrived)s, %(parquet_first_performed)s, %(parquet_last_performed)s, %(parquet_in_process)s, %(parquet_job_id)s, %(flag_for_parquet)s) RETURNING partitions.id]\n[parameters: {'schema_name_id': 352, 'name': 'dt', 'type': 'string', 'value': '2017-06-03', 'location': 's3://sfdl-segment-ssdc-dev/separated_dt/identify/dt=2017-06-03/', 'processed': 0, 'created': '2023-06-09 09:22:53.533938', 'data_first_arrived': '2023-06-09 09:22:53.533938', 'data_last_arrived': '2023-06-09 09:22:53.533938', 'parquet_first_performed': None, 'parquet_last_performed': None, 'parquet_in_process': None, 'parquet_job_id': None, 'flag_for_parquet': True}]\n(Background on this error at: https://sqlalche.me/e/14/gkpj)\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File \"/schema_manager/schema_generator/schema_scanner.py\", line 102, in scan\n    response = service.run()\n  File \"/schema_manager/schema_generator/process_hive_table.py\", line 105, in run\n    response['ingested_partitions'] = pi.run()  # *total* current number of partitions in the table\n  File \"/schema_manager/schema_manager/service/partition_ingester.py\", line 29, in run\n    self.insert_and_clear_partition(schema_version.hive_path, row['partition'])\n  File \"/schema_manager/schema_manager/service/partition_ingester.py\", line 37, in insert_and_clear_partition\n    p = PartitionBagService.backfill_insert(self.schema_name, name, 'string', value, s3_location)\n  File \"/schema_manager/schema_manager/service/partition_bag.py\", line 60, in backfill_insert\n    return PartitionBagService._insert(schema_name, name, type, value, location)\n  File \"/schema_manager/schema_manager/service/partition_bag.py\", line 117, in _insert\n    raise PartitionBagException('failed to insert partition: {}'.format(e))\nschema_manager.service.partition_bag.PartitionBagException: failed to insert partition: (psycopg2.errors.UniqueViolation) duplicate key value violates unique constraint \"partitions_schema_name_id_value_name_key\"\nDETAIL:  Key (schema_name_id, value, name)=(352, 2017-06-03, dt) already exists.\n\n[SQL: INSERT INTO partitions (schema_name_id, name, type, value, location, processed, created, data_first_arrived, data_last_arrived, parquet_first_performed, parquet_last_performed, parquet_in_process, parquet_job_id, flag_for_parquet) VALUES (%(schema_name_id)s, %(name)s, %(type)s, %(value)s, %(location)s, %(processed)s, %(created)s, %(data_first_arrived)s, %(data_last_arrived)s, %(parquet_first_performed)s, %(parquet_last_performed)s, %(parquet_in_process)s, %(parquet_job_id)s, %(flag_for_parquet)s) RETURNING partitions.id]\n[parameters: {'schema_name_id': 352, 'name': 'dt', 'type': 'string', 'value': '2017-06-03', 'location': 's3://sfdl-segment-ssdc-dev/separated_dt/identify/dt=2017-06-03/', 'processed': 0, 'created': '2023-06-09 09:22:53.533938', 'data_first_arrived': '2023-06-09 09:22:53.533938', 'data_last_arrived': '2023-06-09 09:22:53.533938', 'parquet_first_performed': None, 'parquet_last_performed': None, 'parquet_in_process': None, 'parquet_job_id': None, 'flag_for_parquet': True}]\n(Background on this error at: https://sqlalche.me/e/14/gkpj)\n: Expecting value: line 1 column 1 (char 0)"
}

{
    "datatype_dict": {
        "originalTimestamp": "timestamp_iso8601",
        "receivedAt": "timestamp_iso8601",
        "sentAt": "timestamp_iso8601",
        "timestamp": "timestamp_iso8601"
    },
    "file_format": "json",
    "is_schema_datatyped": false,
    "partition": {
        "location": "s3://sfdl-segment-ssdc-dev/separated_dt/identify/dt=2023-06-09/",
        "name": "dt",
        "type": "string",
        "value": "2023-06-09"
    },
    "s3": {
        "bucket": "sfdl-segment-ssdc-dev",
        "hive_path": "s3://sfdl-segment-ssdc-dev/separated_dt/identify",
        "key": "separated_dt/identify/dt=2023-06-09/1686319894293.events-archiver-480-48.8ebb200.abffa4ac-5e46-4031-876a-9482f56f44e5.json.gz"
    },
    "schema": {
        "name": "sfdl_dev_segment.ssdc_identify",
        "type": "segment"
    }
}

{
    "ddl_changed": true,
    "ingested_partitions": 1217,
    "partitions_applied": false,
    "success": true
}

{
    "datatype_dict": {
        "originalTimestamp": "timestamp_iso8601",
        "receivedAt": "timestamp_iso8601",
        "sentAt": "timestamp_iso8601",
        "timestamp": "timestamp_iso8601"
    },
    "file_format": "json",
    "is_schema_datatyped": false,
    "partition": {
        "location": "s3://sfdl-segment-ssdc-dev/separated_dt/identify/dt=2021-07-22/",
        "name": "dt",
        "type": "string",
        "value": "2021-07-22"
    },
    "s3": {
        "bucket": "sfdl-segment-ssdc-dev",
        "hive_path": "s3://sfdl-segment-ssdc-dev/separated_dt/identify",
        "key": "separated_dt/identify/dt=2021-07-22/1626993549210.efda54410a74.8624813.f897e1c0-c9b3-4615-ba8e-b46f3b6835e3.json.gz"
    },
    "schema": {
        "name": "sfdl_dev_segment.ssdc_identify",
        "type": "segment"
    }
}