--- updatedAt: 2025-03-06T23:15:11.000Z agentTools: projectIndex: https://reference.airbyte.com/llms.txt --- # Update a Destination This endpoint can be used to update the configuration of a Destination. Example: If the Destination's configuration JSON contains: ```json json { "start_date": "2020-01-01", "customer_id": "123456789" } ``` You can update with a configuration of: ```json { "customer_id": "abcdefghi" } ``` In this scenario, only the customer\_id will be updated. start\_date will remain the same within the configuration. The other properties in the request body are optional. Any properties not specified in the request body will not be modified from their original values. If you would like to **remove** any properties from your Destination's configuration, this should be done with the [PUT operation](https://reference.airbyte.com/reference/putdestination). # OpenAPI definition ```json { "openapi": "3.1.0", "info": { "title": "Destinations", "version": "1.0.0", "description": "Programatically control Airbyte Cloud, OSS & Enterprise." }, "servers": [ { "url": "https://api.airbyte.com/v1", "description": "Airbyte API v1" } ], "paths": { "/destinations/{destinationId}": { "patch": { "tags": [ "Destinations" ], "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/DestinationPatchRequest" }, "examples": { "Destination Update Request Example": { "value": { "configuration": { "conversion_window_days": 14, "customer_id": "1234567890", "start_date": "2023-01-01", "end_date": "2024-01-01" }, "name": "My Destination" } } } } } }, "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/DestinationResponse" }, "examples": { "Destination Update Response Example": { "value": { "destinationId": "18dccc91-0ab1-4f72-9ed7-0b8fc27c5826", "name": "running", "sourceType": "postgres", "workspaceId": "744cc0ed-7f05-4949-9e60-2a814f90c035", "configuration": { "conversion_window_days": 14, "customer_id": "1234567890", "start_date": "2023-01-01", "end_date": "2024-01-01" } } } } } }, "description": "Update a Destination" }, "403": { "description": "Not allowed" }, "404": { "description": "Not found" } }, "operationId": "patchDestination", "summary": "Update a Destination", "x-use-speakeasy-middleware": true }, "parameters": [ { "name": "destinationId", "schema": { "format": "UUID", "type": "string" }, "in": "path", "required": true } ] } }, "components": { "schemas": { "DestinationResponse": { "title": "Root Type for DestinationResponse", "description": "Provides details of a single destination.", "type": "object", "required": [ "destinationId", "name", "destinationType", "workspaceId", "configuration" ], "properties": { "destinationId": { "format": "UUID", "type": "string" }, "name": { "type": "string" }, "destinationType": { "type": "string" }, "workspaceId": { "format": "UUID", "type": "string" }, "configuration": { "$ref": "#/components/schemas/DestinationConfiguration" } }, "example": { "destinationId": "18dccc91-0ab1-4f72-9ed7-0b8fc27c5826", "name": "Analytics Team Postgres", "destinationType": "postgres", "workspaceId": "871d9b60-11d1-44cb-8c92-c246d53bf87e" } }, "DestinationPatchRequest": { "type": "object", "properties": { "name": { "type": "string" }, "configuration": { "$ref": "#/components/schemas/DestinationConfiguration" } }, "x-implements": "io.airbyte.public_api.server.helpers.ConfigurableActor", "x-speakeasy-entity": "Destination", "x-speakeasy-param-suppress-computed-diff": true }, "destination-gcs": { "title": "GCS Destination Spec", "type": "object", "required": [ "gcs_bucket_name", "gcs_bucket_path", "credential", "format", "destinationType" ], "properties": { "gcs_bucket_name": { "title": "GCS Bucket Name", "order": 1, "type": "string", "description": "You can find the bucket name in the App Engine Admin console Application Settings page, under the label Google Cloud Storage Bucket. Read more here.", "examples": [ "airbyte_sync" ] }, "gcs_bucket_path": { "title": "GCS Bucket Path", "description": "GCS Bucket Path string Subdirectory under the above bucket to sync the data into.", "order": 2, "type": "string", "examples": [ "data_sync/test" ] }, "gcs_bucket_region": { "title": "GCS Bucket Region", "type": "string", "order": 3, "default": "us", "description": "Select a Region of the GCS Bucket. Read more here.", "enum": [ "northamerica-northeast1", "northamerica-northeast2", "us-central1", "us-east1", "us-east4", "us-west1", "us-west2", "us-west3", "us-west4", "southamerica-east1", "southamerica-west1", "europe-central2", "europe-north1", "europe-west1", "europe-west2", "europe-west3", "europe-west4", "europe-west6", "asia-east1", "asia-east2", "asia-northeast1", "asia-northeast2", "asia-northeast3", "asia-south1", "asia-south2", "asia-southeast1", "asia-southeast2", "australia-southeast1", "australia-southeast2", "asia", "eu", "us", "asia1", "eur4", "nam4" ] }, "credential": { "title": "Authentication", "description": "An HMAC key is a type of credential and can be associated with a service account or a user account in Cloud Storage. Read more here.", "type": "object", "order": 0, "oneOf": [ { "title": "HMAC Key", "required": [ "credential_type", "hmac_key_access_id", "hmac_key_secret" ], "properties": { "credential_type": { "type": "string", "enum": [ "HMAC_KEY" ], "default": "HMAC_KEY" }, "hmac_key_access_id": { "type": "string", "description": "When linked to a service account, this ID is 61 characters long; when linked to a user account, it is 24 characters long. Read more here.", "title": "Access ID", "airbyte_secret": true, "order": 0, "examples": [ "1234567890abcdefghij1234" ], "x-speakeasy-param-sensitive": true }, "hmac_key_secret": { "type": "string", "description": "The corresponding secret for the access ID. It is a 40-character base-64 encoded string. Read more here.", "title": "Secret", "airbyte_secret": true, "order": 1, "examples": [ "1234567890abcdefghij1234567890ABCDEFGHIJ" ], "x-speakeasy-param-sensitive": true } } } ] }, "format": { "title": "Output Format", "type": "object", "description": "Output data format. One of the following formats must be selected - AVRO format, PARQUET format, CSV format, or JSONL format.", "order": 4, "oneOf": [ { "title": "Avro: Apache Avro", "required": [ "format_type", "compression_codec" ], "properties": { "format_type": { "type": "string", "enum": [ "Avro" ], "default": "Avro" }, "compression_codec": { "title": "Compression Codec", "description": "The compression algorithm used to compress data. Default to no compression.", "type": "object", "oneOf": [ { "title": "No Compression", "required": [ "codec" ], "properties": { "codec": { "type": "string", "enum": [ "no compression" ], "default": "no compression" } } }, { "title": "Deflate", "required": [ "codec" ], "properties": { "codec": { "type": "string", "enum": [ "Deflate" ], "default": "Deflate" }, "compression_level": { "title": "Deflate level", "description": "0: no compression & fastest, 9: best compression & slowest.", "type": "integer", "default": 0, "minimum": 0, "maximum": 9 } } }, { "title": "bzip2", "required": [ "codec" ], "properties": { "codec": { "type": "string", "enum": [ "bzip2" ], "default": "bzip2" } } }, { "title": "xz", "required": [ "codec" ], "properties": { "codec": { "type": "string", "enum": [ "xz" ], "default": "xz" }, "compression_level": { "title": "Compression Level", "description": "The presets 0-3 are fast presets with medium compression. The presets 4-6 are fairly slow presets with high compression. The default preset is 6. The presets 7-9 are like the preset 6 but use bigger dictionaries and have higher compressor and decompressor memory requirements. Unless the uncompressed size of the file exceeds 8 MiB, 16 MiB, or 32 MiB, it is waste of memory to use the presets 7, 8, or 9, respectively. Read more here for details.", "type": "integer", "default": 6, "minimum": 0, "maximum": 9 } } }, { "title": "zstandard", "required": [ "codec" ], "properties": { "codec": { "type": "string", "enum": [ "zstandard" ], "default": "zstandard" }, "compression_level": { "title": "Compression Level", "description": "Negative levels are 'fast' modes akin to lz4 or snappy, levels above 9 are generally for archival purposes, and levels above 18 use a lot of memory.", "type": "integer", "default": 3, "minimum": -5, "maximum": 22 }, "include_checksum": { "title": "Include Checksum", "description": "If true, include a checksum with each data block.", "type": "boolean", "default": false } } }, { "title": "snappy", "required": [ "codec" ], "properties": { "codec": { "type": "string", "enum": [ "snappy" ], "default": "snappy" } } } ] } } }, { "title": "CSV: Comma-Separated Values", "required": [ "format_type" ], "properties": { "format_type": { "type": "string", "enum": [ "CSV" ], "default": "CSV" }, "flattening": { "type": "string", "title": "Normalization", "description": "Whether the input JSON data should be normalized (flattened) in the output CSV. Please refer to docs for details.", "default": "No flattening", "enum": [ "No flattening", "Root level flattening" ] }, "compression": { "title": "Compression", "type": "object", "description": "Whether the output files should be compressed. If compression is selected, the output filename will have an extra extension (GZIP: \".csv.gz\").", "oneOf": [ { "title": "No Compression", "requires": [ "compression_type" ], "properties": { "compression_type": { "type": "string", "enum": [ "No Compression" ], "default": "No Compression" } } }, { "title": "GZIP", "requires": [ "compression_type" ], "properties": { "compression_type": { "type": "string", "enum": [ "GZIP" ], "default": "GZIP" } } } ] } } }, { "title": "JSON Lines: newline-delimited JSON", "required": [ "format_type" ], "properties": { "format_type": { "type": "string", "enum": [ "JSONL" ], "default": "JSONL" }, "compression": { "title": "Compression", "type": "object", "description": "Whether the output files should be compressed. If compression is selected, the output filename will have an extra extension (GZIP: \".jsonl.gz\").", "oneOf": [ { "title": "No Compression", "requires": "compression_type", "properties": { "compression_type": { "type": "string", "enum": [ "No Compression" ], "default": "No Compression" } } }, { "title": "GZIP", "requires": "compression_type", "properties": { "compression_type": { "type": "string", "enum": [ "GZIP" ], "default": "GZIP" } } } ] } } }, { "title": "Parquet: Columnar Storage", "required": [ "format_type" ], "properties": { "format_type": { "type": "string", "enum": [ "Parquet" ], "default": "Parquet" }, "compression_codec": { "title": "Compression Codec", "description": "The compression algorithm used to compress data pages.", "type": "string", "default": "UNCOMPRESSED", "enum": [ "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "BROTLI", "LZ4", "ZSTD" ] }, "block_size_mb": { "title": "Block Size (Row Group Size) (MB)", "description": "This is the size of a row group being buffered in memory. It limits the memory usage when writing. Larger values will improve the IO when reading, but consume more memory when writing. Default: 128 MB.", "type": "integer", "default": 128, "examples": [ 128 ] }, "max_padding_size_mb": { "title": "Max Padding Size (MB)", "description": "Maximum size allowed as padding to align row groups. This is also the minimum size of a row group. Default: 8 MB.", "type": "integer", "default": 8, "examples": [ 8 ] }, "page_size_kb": { "title": "Page Size (KB)", "description": "The page size is for compression. A block is composed of pages. A page is the smallest unit that must be read fully to access a single record. If this value is too small, the compression will deteriorate. Default: 1024 KB.", "type": "integer", "default": 1024, "examples": [ 1024 ] }, "dictionary_page_size_kb": { "title": "Dictionary Page Size (KB)", "description": "There is one dictionary page per column per row group when dictionary encoding is used. The dictionary page size works like the page size but for dictionary. Default: 1024 KB.", "type": "integer", "default": 1024, "examples": [ 1024 ] }, "dictionary_encoding": { "title": "Dictionary Encoding", "description": "Default: true.", "type": "boolean", "default": true } } } ] }, "destinationType": { "title": "gcs", "const": "gcs", "enum": [ "gcs" ], "order": 0, "type": "string" } } }, "destination-xata": { "title": "Destination Xata", "type": "object", "required": [ "api_key", "db_url", "destinationType" ], "properties": { "api_key": { "title": "API Key", "description": "API Key to connect.", "type": "string", "order": 0, "airbyte_secret": true, "x-speakeasy-param-sensitive": true }, "db_url": { "title": "Database URL", "description": "URL pointing to your workspace.", "type": "string", "order": 1, "example": "https://my-workspace-abc123.us-east-1.xata.sh/db/nyc-taxi-fares:main" }, "destinationType": { "title": "xata", "const": "xata", "enum": [ "xata" ], "order": 0, "type": "string" } } }, "destination-clickhouse": { "title": "ClickHouse Destination Spec", "type": "object", "required": [ "host", "port", "database", "username", "destinationType" ], "properties": { "host": { "title": "Host", "description": "Hostname of the database.", "type": "string", "order": 0 }, "port": { "title": "Port", "description": "HTTP port of the database.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 8123, "examples": [ "8123" ], "order": 1 }, "database": { "title": "DB Name", "description": "Name of the database.", "type": "string", "order": 2 }, "username": { "title": "User", "description": "Username to use to access the database.", "type": "string", "order": 3 }, "password": { "title": "Password", "description": "Password associated with the username.", "type": "string", "airbyte_secret": true, "order": 4, "x-speakeasy-param-sensitive": true }, "jdbc_url_params": { "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "title": "JDBC URL Params", "type": "string", "order": 5 }, "tunnel_method": { "type": "object", "title": "SSH Tunnel Method", "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", "oneOf": [ { "title": "No Tunnel", "required": [ "tunnel_method" ], "properties": { "tunnel_method": { "description": "No ssh tunnel needed to connect to database", "type": "string", "const": "NO_TUNNEL", "order": 0, "enum": [ "NO_TUNNEL" ] } } }, { "title": "SSH Key Authentication", "required": [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "ssh_key" ], "properties": { "tunnel_method": { "description": "Connect through a jump server tunnel host using username and ssh key", "type": "string", "const": "SSH_KEY_AUTH", "order": 0, "enum": [ "SSH_KEY_AUTH" ] }, "tunnel_host": { "title": "SSH Tunnel Jump Server Host", "description": "Hostname of the jump server host that allows inbound ssh tunnel.", "type": "string", "order": 1 }, "tunnel_port": { "title": "SSH Connection Port", "description": "Port on the proxy/jump server that accepts inbound ssh connections.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 22, "examples": [ "22" ], "order": 2 }, "tunnel_user": { "title": "SSH Login Username", "description": "OS-level username for logging into the jump server host.", "type": "string", "order": 3 }, "ssh_key": { "title": "SSH Private Key", "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", "type": "string", "airbyte_secret": true, "multiline": true, "order": 4, "x-speakeasy-param-sensitive": true } } }, { "title": "Password Authentication", "required": [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "tunnel_user_password" ], "properties": { "tunnel_method": { "description": "Connect through a jump server tunnel host using username and password authentication", "type": "string", "const": "SSH_PASSWORD_AUTH", "order": 0, "enum": [ "SSH_PASSWORD_AUTH" ] }, "tunnel_host": { "title": "SSH Tunnel Jump Server Host", "description": "Hostname of the jump server host that allows inbound ssh tunnel.", "type": "string", "order": 1 }, "tunnel_port": { "title": "SSH Connection Port", "description": "Port on the proxy/jump server that accepts inbound ssh connections.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 22, "examples": [ "22" ], "order": 2 }, "tunnel_user": { "title": "SSH Login Username", "description": "OS-level username for logging into the jump server host", "type": "string", "order": 3 }, "tunnel_user_password": { "title": "Password", "description": "OS-level password for logging into the jump server host", "type": "string", "airbyte_secret": true, "order": 4, "x-speakeasy-param-sensitive": true } } } ] }, "destinationType": { "title": "clickhouse", "const": "clickhouse", "enum": [ "clickhouse" ], "order": 0, "type": "string" } } }, "destination-mssql": { "title": "MS SQL Server Destination Spec", "type": "object", "required": [ "host", "port", "username", "database", "schema", "destinationType" ], "properties": { "host": { "title": "Host", "description": "The host name of the MSSQL database.", "type": "string", "order": 0 }, "port": { "title": "Port", "description": "The port of the MSSQL database.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 1433, "examples": [ "1433" ], "order": 1 }, "database": { "title": "DB Name", "description": "The name of the MSSQL database.", "type": "string", "order": 2 }, "schema": { "title": "Default Schema", "description": "The default schema tables are written to if the source does not specify a namespace. The usual value for this field is \"public\".", "type": "string", "examples": [ "public" ], "default": "public", "order": 3 }, "username": { "title": "User", "description": "The username which is used to access the database.", "type": "string", "order": 4 }, "password": { "title": "Password", "description": "The password associated with this username.", "type": "string", "airbyte_secret": true, "order": 5, "x-speakeasy-param-sensitive": true }, "jdbc_url_params": { "title": "JDBC URL Params", "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "type": "string", "order": 6 }, "ssl_method": { "title": "SSL Method", "type": "object", "description": "The encryption method which is used to communicate with the database.", "order": 7, "oneOf": [ { "title": "Encrypted (trust server certificate)", "description": "Use the certificate provided by the server without verification. (For testing purposes only!)", "required": [ "ssl_method" ], "type": "object", "properties": { "ssl_method": { "type": "string", "const": "encrypted_trust_server_certificate", "enum": [ "encrypted_trust_server_certificate" ], "default": "encrypted_trust_server_certificate" } } }, { "title": "Encrypted (verify certificate)", "description": "Verify and use the certificate provided by the server.", "required": [ "ssl_method", "trustStoreName", "trustStorePassword" ], "type": "object", "properties": { "ssl_method": { "type": "string", "const": "encrypted_verify_certificate", "enum": [ "encrypted_verify_certificate" ], "default": "encrypted_verify_certificate" }, "hostNameInCertificate": { "title": "Host Name In Certificate", "type": "string", "description": "Specifies the host name of the server. The value of this property must match the subject property of the certificate.", "order": 8 } } } ] }, "tunnel_method": { "type": "object", "title": "SSH Tunnel Method", "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", "oneOf": [ { "title": "No Tunnel", "required": [ "tunnel_method" ], "properties": { "tunnel_method": { "description": "No ssh tunnel needed to connect to database", "type": "string", "const": "NO_TUNNEL", "order": 0, "enum": [ "NO_TUNNEL" ] } } }, { "title": "SSH Key Authentication", "required": [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "ssh_key" ], "properties": { "tunnel_method": { "description": "Connect through a jump server tunnel host using username and ssh key", "type": "string", "const": "SSH_KEY_AUTH", "order": 0, "enum": [ "SSH_KEY_AUTH" ] }, "tunnel_host": { "title": "SSH Tunnel Jump Server Host", "description": "Hostname of the jump server host that allows inbound ssh tunnel.", "type": "string", "order": 1 }, "tunnel_port": { "title": "SSH Connection Port", "description": "Port on the proxy/jump server that accepts inbound ssh connections.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 22, "examples": [ "22" ], "order": 2 }, "tunnel_user": { "title": "SSH Login Username", "description": "OS-level username for logging into the jump server host.", "type": "string", "order": 3 }, "ssh_key": { "title": "SSH Private Key", "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", "type": "string", "airbyte_secret": true, "multiline": true, "order": 4, "x-speakeasy-param-sensitive": true } } }, { "title": "Password Authentication", "required": [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "tunnel_user_password" ], "properties": { "tunnel_method": { "description": "Connect through a jump server tunnel host using username and password authentication", "type": "string", "const": "SSH_PASSWORD_AUTH", "order": 0, "enum": [ "SSH_PASSWORD_AUTH" ] }, "tunnel_host": { "title": "SSH Tunnel Jump Server Host", "description": "Hostname of the jump server host that allows inbound ssh tunnel.", "type": "string", "order": 1 }, "tunnel_port": { "title": "SSH Connection Port", "description": "Port on the proxy/jump server that accepts inbound ssh connections.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 22, "examples": [ "22" ], "order": 2 }, "tunnel_user": { "title": "SSH Login Username", "description": "OS-level username for logging into the jump server host", "type": "string", "order": 3 }, "tunnel_user_password": { "title": "Password", "description": "OS-level password for logging into the jump server host", "type": "string", "airbyte_secret": true, "order": 4, "x-speakeasy-param-sensitive": true } } } ] }, "destinationType": { "title": "mssql", "const": "mssql", "enum": [ "mssql" ], "order": 0, "type": "string" } } }, "destination-mysql": { "title": "MySQL Destination Spec", "type": "object", "required": [ "host", "port", "username", "database", "destinationType" ], "properties": { "host": { "title": "Host", "description": "Hostname of the database.", "type": "string", "order": 0 }, "port": { "title": "Port", "description": "Port of the database.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 3306, "examples": [ "3306" ], "order": 1 }, "database": { "title": "DB Name", "description": "Name of the database.", "type": "string", "order": 2 }, "username": { "title": "User", "description": "Username to use to access the database.", "type": "string", "order": 3 }, "password": { "title": "Password", "description": "Password associated with the username.", "type": "string", "airbyte_secret": true, "order": 4, "x-speakeasy-param-sensitive": true }, "jdbc_url_params": { "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "title": "JDBC URL Params", "type": "string", "order": 6 }, "tunnel_method": { "type": "object", "title": "SSH Tunnel Method", "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", "oneOf": [ { "title": "No Tunnel", "required": [ "tunnel_method" ], "properties": { "tunnel_method": { "description": "No ssh tunnel needed to connect to database", "type": "string", "const": "NO_TUNNEL", "order": 0, "enum": [ "NO_TUNNEL" ] } } }, { "title": "SSH Key Authentication", "required": [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "ssh_key" ], "properties": { "tunnel_method": { "description": "Connect through a jump server tunnel host using username and ssh key", "type": "string", "const": "SSH_KEY_AUTH", "order": 0, "enum": [ "SSH_KEY_AUTH" ] }, "tunnel_host": { "title": "SSH Tunnel Jump Server Host", "description": "Hostname of the jump server host that allows inbound ssh tunnel.", "type": "string", "order": 1 }, "tunnel_port": { "title": "SSH Connection Port", "description": "Port on the proxy/jump server that accepts inbound ssh connections.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 22, "examples": [ "22" ], "order": 2 }, "tunnel_user": { "title": "SSH Login Username", "description": "OS-level username for logging into the jump server host.", "type": "string", "order": 3 }, "ssh_key": { "title": "SSH Private Key", "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", "type": "string", "airbyte_secret": true, "multiline": true, "order": 4, "x-speakeasy-param-sensitive": true } } }, { "title": "Password Authentication", "required": [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "tunnel_user_password" ], "properties": { "tunnel_method": { "description": "Connect through a jump server tunnel host using username and password authentication", "type": "string", "const": "SSH_PASSWORD_AUTH", "order": 0, "enum": [ "SSH_PASSWORD_AUTH" ] }, "tunnel_host": { "title": "SSH Tunnel Jump Server Host", "description": "Hostname of the jump server host that allows inbound ssh tunnel.", "type": "string", "order": 1 }, "tunnel_port": { "title": "SSH Connection Port", "description": "Port on the proxy/jump server that accepts inbound ssh connections.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 22, "examples": [ "22" ], "order": 2 }, "tunnel_user": { "title": "SSH Login Username", "description": "OS-level username for logging into the jump server host", "type": "string", "order": 3 }, "tunnel_user_password": { "title": "Password", "description": "OS-level password for logging into the jump server host", "type": "string", "airbyte_secret": true, "order": 4, "x-speakeasy-param-sensitive": true } } } ] }, "destinationType": { "title": "mysql", "const": "mysql", "enum": [ "mysql" ], "order": 0, "type": "string" } } }, "destination-pubsub": { "title": "Google PubSub Destination Spec", "type": "object", "required": [ "project_id", "topic_id", "credentials_json", "ordering_enabled", "batching_enabled", "destinationType" ], "properties": { "project_id": { "type": "string", "description": "The GCP project ID for the project containing the target PubSub.", "title": "Project ID" }, "topic_id": { "type": "string", "description": "The PubSub topic ID in the given GCP project ID.", "title": "PubSub Topic ID" }, "credentials_json": { "type": "string", "description": "The contents of the JSON service account key. Check out the docs if you need help generating this key.", "title": "Credentials JSON", "airbyte_secret": true }, "ordering_enabled": { "title": "Message Ordering Enabled", "description": "If TRUE PubSub publisher will have message ordering enabled. Every message will have an ordering key of stream", "type": "boolean", "default": false }, "batching_enabled": { "type": "boolean", "title": "Message Batching Enabled", "description": "If TRUE messages will be buffered instead of sending them one by one", "default": false }, "batching_delay_threshold": { "type": "integer", "title": "Message Batching: Delay Threshold", "description": "Number of ms before the buffer is flushed", "default": 1, "minimum": 1 }, "batching_element_count_threshold": { "type": "integer", "title": "Message Batching: Element Count Threshold", "description": "Number of messages before the buffer is flushed", "default": 1, "minimum": 1 }, "batching_request_bytes_threshold": { "type": "integer", "title": "Message Batching: Request Bytes Threshold", "description": "Number of bytes before the buffer is flushed", "default": 1, "minimum": 1 }, "destinationType": { "title": "pubsub", "const": "pubsub", "enum": [ "pubsub" ], "order": 0, "type": "string" } } }, "destination-weaviate": { "title": "Destination Config", "description": "The configuration model for the Vector DB based destinations. This model is used to generate the UI for the destination configuration,\nas well as to provide type safety for the configuration passed to the destination.\n\nThe configuration model is composed of four parts:\n* Processing configuration\n* Embedding configuration\n* Indexing configuration\n* Advanced configuration\n\nProcessing, embedding and advanced configuration are provided by this base class, while the indexing configuration is provided by the destination connector in the sub class.", "type": "object", "properties": { "embedding": { "title": "Embedding", "description": "Embedding configuration", "group": "embedding", "type": "object", "oneOf": [ { "title": "No external embedding", "type": "object", "properties": { "mode": { "title": "Mode", "default": "no_embedding", "const": "no_embedding", "enum": [ "no_embedding" ], "type": "string" } }, "description": "Do not calculate and pass embeddings to Weaviate. Suitable for clusters with configured vectorizers to calculate embeddings within Weaviate or for classes that should only support regular text search.", "required": [ "mode" ] }, { "title": "Azure OpenAI", "type": "object", "properties": { "mode": { "title": "Mode", "default": "azure_openai", "const": "azure_openai", "enum": [ "azure_openai" ], "type": "string" }, "openai_key": { "title": "Azure OpenAI API key", "description": "The API key for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true }, "api_base": { "title": "Resource base URL", "description": "The base URL for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", "examples": [ "https://your-resource-name.openai.azure.com" ], "type": "string" }, "deployment": { "title": "Deployment", "description": "The deployment for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", "examples": [ "your-resource-name" ], "type": "string" } }, "required": [ "openai_key", "api_base", "deployment", "mode" ], "description": "Use the Azure-hosted OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." }, { "title": "OpenAI", "type": "object", "properties": { "mode": { "title": "Mode", "default": "openai", "const": "openai", "enum": [ "openai" ], "type": "string" }, "openai_key": { "title": "OpenAI API key", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true } }, "required": [ "openai_key", "mode" ], "description": "Use the OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." }, { "title": "Cohere", "type": "object", "properties": { "mode": { "title": "Mode", "default": "cohere", "const": "cohere", "enum": [ "cohere" ], "type": "string" }, "cohere_key": { "title": "Cohere API key", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true } }, "required": [ "cohere_key", "mode" ], "description": "Use the Cohere API to embed text." }, { "title": "From Field", "type": "object", "properties": { "mode": { "title": "Mode", "default": "from_field", "const": "from_field", "enum": [ "from_field" ], "type": "string" }, "field_name": { "title": "Field name", "description": "Name of the field in the record that contains the embedding", "examples": [ "embedding", "vector" ], "type": "string" }, "dimensions": { "title": "Embedding dimensions", "description": "The number of dimensions the embedding model is generating", "examples": [ 1536, 384 ], "type": "integer" } }, "required": [ "field_name", "dimensions", "mode" ], "description": "Use a field in the record as the embedding. This is useful if you already have an embedding for your data and want to store it in the vector store." }, { "title": "Fake", "type": "object", "properties": { "mode": { "title": "Mode", "default": "fake", "const": "fake", "enum": [ "fake" ], "type": "string" } }, "description": "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs.", "required": [ "mode" ] }, { "title": "OpenAI-compatible", "type": "object", "properties": { "mode": { "title": "Mode", "default": "openai_compatible", "const": "openai_compatible", "enum": [ "openai_compatible" ], "type": "string" }, "api_key": { "title": "API key", "default": "", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true }, "base_url": { "title": "Base URL", "description": "The base URL for your OpenAI-compatible service", "examples": [ "https://your-service-name.com" ], "type": "string" }, "model_name": { "title": "Model name", "description": "The name of the model to use for embedding", "default": "text-embedding-ada-002", "examples": [ "text-embedding-ada-002" ], "type": "string" }, "dimensions": { "title": "Embedding dimensions", "description": "The number of dimensions the embedding model is generating", "examples": [ 1536, 384 ], "type": "integer" } }, "required": [ "base_url", "dimensions", "mode" ], "description": "Use a service that's compatible with the OpenAI API to embed text." } ] }, "processing": { "title": "ProcessingConfigModel", "type": "object", "properties": { "chunk_size": { "title": "Chunk size", "description": "Size of chunks in tokens to store in vector store (make sure it is not too big for the context if your LLM)", "maximum": 8191, "minimum": 1, "type": "integer" }, "chunk_overlap": { "title": "Chunk overlap", "description": "Size of overlap between chunks in tokens to store in vector store to better capture relevant context", "default": 0, "type": "integer" }, "text_fields": { "title": "Text fields to embed", "description": "List of fields in the record that should be used to calculate the embedding. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered text fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.", "default": [], "always_show": true, "examples": [ "text", "user.name", "users.*.name" ], "type": "array", "items": { "type": "string" } }, "metadata_fields": { "title": "Fields to store as metadata", "description": "List of fields in the record that should be stored as metadata. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered metadata fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. When specifying nested paths, all matching values are flattened into an array set to a field named by the path.", "default": [], "always_show": true, "examples": [ "age", "user", "user.name" ], "type": "array", "items": { "type": "string" } }, "text_splitter": { "title": "Text splitter", "description": "Split text fields into chunks based on the specified method.", "type": "object", "oneOf": [ { "title": "By Separator", "type": "object", "properties": { "mode": { "title": "Mode", "default": "separator", "const": "separator", "enum": [ "separator" ], "type": "string" }, "separators": { "title": "Separators", "description": "List of separator strings to split text fields by. The separator itself needs to be wrapped in double quotes, e.g. to split by the dot character, use \".\". To split by a newline, use \"\\n\".", "default": [ "\"\\n\\n\"", "\"\\n\"", "\" \"", "\"\"" ], "type": "array", "items": { "type": "string" } }, "keep_separator": { "title": "Keep separator", "description": "Whether to keep the separator in the resulting chunks", "default": false, "type": "boolean" } }, "description": "Split the text by the list of separators until the chunk size is reached, using the earlier mentioned separators where possible. This is useful for splitting text fields by paragraphs, sentences, words, etc.", "required": [ "mode" ] }, { "title": "By Markdown header", "type": "object", "properties": { "mode": { "title": "Mode", "default": "markdown", "const": "markdown", "enum": [ "markdown" ], "type": "string" }, "split_level": { "title": "Split level", "description": "Level of markdown headers to split text fields by. Headings down to the specified level will be used as split points", "default": 1, "minimum": 1, "maximum": 6, "type": "integer" } }, "description": "Split the text by Markdown headers down to the specified header level. If the chunk size fits multiple sections, they will be combined into a single chunk.", "required": [ "mode" ] }, { "title": "By Programming Language", "type": "object", "properties": { "mode": { "title": "Mode", "default": "code", "const": "code", "enum": [ "code" ], "type": "string" }, "language": { "title": "Language", "description": "Split code in suitable places based on the programming language", "enum": [ "cpp", "go", "java", "js", "php", "proto", "python", "rst", "ruby", "rust", "scala", "swift", "markdown", "latex", "html", "sol" ], "type": "string" } }, "required": [ "language", "mode" ], "description": "Split the text by suitable delimiters based on the programming language. This is useful for splitting code into chunks." } ] }, "field_name_mappings": { "title": "Field name mappings", "description": "List of fields to rename. Not applicable for nested fields, but can be used to rename fields already flattened via dot notation.", "default": [], "type": "array", "items": { "title": "FieldNameMappingConfigModel", "type": "object", "properties": { "from_field": { "title": "From field name", "description": "The field name in the source", "type": "string" }, "to_field": { "title": "To field name", "description": "The field name to use in the destination", "type": "string" } }, "required": [ "from_field", "to_field" ] } } }, "required": [ "chunk_size" ], "group": "processing" }, "omit_raw_text": { "title": "Do not store raw text", "description": "Do not store the text that gets embedded along with the vector and the metadata in the destination. If set to true, only the vector and the metadata will be stored - in this case raw text for LLM use cases needs to be retrieved from another source.", "default": false, "group": "advanced", "type": "boolean" }, "indexing": { "title": "Indexing", "type": "object", "properties": { "host": { "title": "Public Endpoint", "description": "The public endpoint of the Weaviate cluster.", "order": 1, "examples": [ "https://my-cluster.weaviate.network" ], "type": "string" }, "auth": { "title": "Authentication", "description": "Authentication method", "type": "object", "order": 2, "oneOf": [ { "title": "API Token", "type": "object", "properties": { "mode": { "title": "Mode", "default": "token", "const": "token", "enum": [ "token" ], "type": "string" }, "token": { "title": "API Token", "description": "API Token for the Weaviate instance", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true } }, "required": [ "token", "mode" ], "description": "Authenticate using an API token (suitable for Weaviate Cloud)" }, { "title": "Username/Password", "type": "object", "properties": { "mode": { "title": "Mode", "default": "username_password", "const": "username_password", "enum": [ "username_password" ], "type": "string" }, "username": { "title": "Username", "description": "Username for the Weaviate cluster", "order": 1, "type": "string" }, "password": { "title": "Password", "description": "Password for the Weaviate cluster", "airbyte_secret": true, "order": 2, "type": "string", "x-speakeasy-param-sensitive": true } }, "required": [ "username", "password", "mode" ], "description": "Authenticate using username and password (suitable for self-managed Weaviate clusters)" }, { "title": "No Authentication", "type": "object", "properties": { "mode": { "title": "Mode", "default": "no_auth", "const": "no_auth", "enum": [ "no_auth" ], "type": "string" } }, "description": "Do not authenticate (suitable for locally running test clusters, do not use for clusters with public IP addresses)", "required": [ "mode" ] } ] }, "batch_size": { "title": "Batch Size", "description": "The number of records to send to Weaviate in each batch", "default": 128, "type": "integer" }, "text_field": { "title": "Text Field", "description": "The field in the object that contains the embedded text", "default": "text", "type": "string" }, "tenant_id": { "title": "Tenant ID", "description": "The tenant ID to use for multi tenancy", "default": "", "airbyte_secret": true, "type": "string" }, "default_vectorizer": { "title": "Default Vectorizer", "description": "The vectorizer to use if new classes need to be created", "default": "none", "enum": [ "none", "text2vec-cohere", "text2vec-huggingface", "text2vec-openai", "text2vec-palm", "text2vec-contextionary", "text2vec-transformers", "text2vec-gpt4all" ], "type": "string" }, "additional_headers": { "title": "Additional headers", "description": "Additional HTTP headers to send with every request.", "default": [], "examples": [ { "header_key": "X-OpenAI-Api-Key", "value": "my-openai-api-key" } ], "type": "array", "items": { "title": "Header", "type": "object", "properties": { "header_key": { "title": "Header Key", "type": "string", "x-speakeasy-param-sensitive": true }, "value": { "title": "Header Value", "airbyte_secret": true, "type": "string" } }, "required": [ "header_key", "value" ] } } }, "required": [ "host", "auth" ], "group": "indexing", "description": "Indexing configuration" }, "destinationType": { "title": "weaviate", "const": "weaviate", "enum": [ "weaviate" ], "order": 0, "type": "string" } }, "required": [ "embedding", "processing", "indexing", "destinationType" ], "groups": [ { "id": "processing", "title": "Processing" }, { "id": "embedding", "title": "Embedding" }, { "id": "indexing", "title": "Indexing" }, { "id": "advanced", "title": "Advanced" } ] }, "destination-keen": { "title": "Keen Spec", "type": "object", "required": [ "project_id", "api_key", "destinationType" ], "properties": { "project_id": { "description": "To get Keen Project ID, navigate to the Access tab from the left-hand, side panel and check the Project Details section.", "title": "Project ID", "type": "string", "examples": [ "58b4acc22ba938934e888322e" ] }, "api_key": { "title": "API Key", "description": "To get Keen Master API Key, navigate to the Access tab from the left-hand, side panel and check the Project Details section.", "type": "string", "examples": [ "ABCDEFGHIJKLMNOPRSTUWXYZ" ], "airbyte_secret": true, "x-speakeasy-param-sensitive": true }, "infer_timestamp": { "title": "Infer Timestamp", "description": "Allow connector to guess keen.timestamp value based on the streamed data.", "type": "boolean", "default": true }, "destinationType": { "title": "keen", "const": "keen", "enum": [ "keen" ], "order": 0, "type": "string" } } }, "destination-mongodb": { "title": "MongoDB Destination Spec", "type": "object", "required": [ "database", "auth_type", "destinationType" ], "properties": { "instance_type": { "description": "MongoDb instance to connect to. For MongoDB Atlas and Replica Set TLS connection is used by default.", "title": "MongoDb Instance Type", "type": "object", "order": 0, "oneOf": [ { "title": "Standalone MongoDb Instance", "required": [ "instance", "host", "port" ], "properties": { "instance": { "type": "string", "enum": [ "standalone" ], "default": "standalone" }, "host": { "title": "Host", "type": "string", "description": "The Host of a Mongo database to be replicated.", "order": 0 }, "port": { "title": "Port", "type": "integer", "description": "The Port of a Mongo database to be replicated.", "minimum": 0, "maximum": 65536, "default": 27017, "examples": [ "27017" ], "order": 1 } } }, { "title": "Replica Set", "required": [ "instance", "server_addresses" ], "properties": { "instance": { "type": "string", "enum": [ "replica" ], "default": "replica" }, "server_addresses": { "title": "Server addresses", "type": "string", "description": "The members of a replica set. Please specify `host`:`port` of each member seperated by comma.", "examples": [ "host1:27017,host2:27017,host3:27017" ], "order": 0 }, "replica_set": { "title": "Replica Set", "type": "string", "description": "A replica set name.", "order": 1 } } }, { "title": "MongoDB Atlas", "required": [ "instance", "cluster_url" ], "properties": { "instance": { "type": "string", "enum": [ "atlas" ], "default": "atlas" }, "cluster_url": { "title": "Cluster URL", "type": "string", "description": "URL of a cluster to connect to.", "order": 0 } } } ] }, "database": { "title": "DB Name", "description": "Name of the database.", "type": "string", "order": 2 }, "auth_type": { "title": "Authorization type", "type": "object", "description": "Authorization type.", "oneOf": [ { "title": "None", "description": "None.", "required": [ "authorization" ], "type": "object", "properties": { "authorization": { "type": "string", "const": "none", "enum": [ "none" ] } } }, { "title": "Login/Password", "description": "Login/Password.", "required": [ "authorization", "username", "password" ], "type": "object", "properties": { "authorization": { "type": "string", "const": "login/password", "enum": [ "login/password" ] }, "username": { "title": "User", "description": "Username to use to access the database.", "type": "string", "order": 1 }, "password": { "title": "Password", "description": "Password associated with the username.", "type": "string", "airbyte_secret": true, "order": 2, "x-speakeasy-param-sensitive": true } } } ] }, "tunnel_method": { "type": "object", "title": "SSH Tunnel Method", "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", "oneOf": [ { "title": "No Tunnel", "required": [ "tunnel_method" ], "properties": { "tunnel_method": { "description": "No ssh tunnel needed to connect to database", "type": "string", "const": "NO_TUNNEL", "order": 0, "enum": [ "NO_TUNNEL" ] } } }, { "title": "SSH Key Authentication", "required": [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "ssh_key" ], "properties": { "tunnel_method": { "description": "Connect through a jump server tunnel host using username and ssh key", "type": "string", "const": "SSH_KEY_AUTH", "order": 0, "enum": [ "SSH_KEY_AUTH" ] }, "tunnel_host": { "title": "SSH Tunnel Jump Server Host", "description": "Hostname of the jump server host that allows inbound ssh tunnel.", "type": "string", "order": 1 }, "tunnel_port": { "title": "SSH Connection Port", "description": "Port on the proxy/jump server that accepts inbound ssh connections.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 22, "examples": [ "22" ], "order": 2 }, "tunnel_user": { "title": "SSH Login Username", "description": "OS-level username for logging into the jump server host.", "type": "string", "order": 3 }, "ssh_key": { "title": "SSH Private Key", "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", "type": "string", "airbyte_secret": true, "multiline": true, "order": 4, "x-speakeasy-param-sensitive": true } } }, { "title": "Password Authentication", "required": [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "tunnel_user_password" ], "properties": { "tunnel_method": { "description": "Connect through a jump server tunnel host using username and password authentication", "type": "string", "const": "SSH_PASSWORD_AUTH", "order": 0, "enum": [ "SSH_PASSWORD_AUTH" ] }, "tunnel_host": { "title": "SSH Tunnel Jump Server Host", "description": "Hostname of the jump server host that allows inbound ssh tunnel.", "type": "string", "order": 1 }, "tunnel_port": { "title": "SSH Connection Port", "description": "Port on the proxy/jump server that accepts inbound ssh connections.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 22, "examples": [ "22" ], "order": 2 }, "tunnel_user": { "title": "SSH Login Username", "description": "OS-level username for logging into the jump server host", "type": "string", "order": 3 }, "tunnel_user_password": { "title": "Password", "description": "OS-level password for logging into the jump server host", "type": "string", "airbyte_secret": true, "order": 4, "x-speakeasy-param-sensitive": true } } } ] }, "destinationType": { "title": "mongodb", "const": "mongodb", "enum": [ "mongodb" ], "order": 0, "type": "string" } } }, "destination-vectara": { "title": "Vectara Config", "type": "object", "properties": { "oauth2": { "title": "OAuth2.0 Credentials", "type": "object", "properties": { "client_id": { "title": "OAuth Client ID", "description": "OAuth2.0 client id", "order": 0, "type": "string" }, "client_secret": { "title": "OAuth Client Secret", "description": "OAuth2.0 client secret", "airbyte_secret": true, "order": 1, "type": "string" } }, "required": [ "client_id", "client_secret" ], "description": "OAuth2.0 credentials used to authenticate admin actions (creating/deleting corpora)", "group": "auth" }, "customer_id": { "title": "Customer ID", "description": "Your customer id as it is in the authenticaion url", "order": 2, "group": "account", "type": "string" }, "corpus_name": { "title": "Corpus Name", "description": "The Name of Corpus to load data into", "order": 3, "group": "account", "type": "string" }, "parallelize": { "title": "Parallelize", "description": "Parallelize indexing into Vectara with multiple threads", "default": false, "always_show": true, "group": "account", "type": "boolean" }, "text_fields": { "title": "Text fields to index with Vectara", "description": "List of fields in the record that should be in the section of the document. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered text fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.", "default": [], "always_show": true, "examples": [ "text", "user.name", "users.*.name" ], "type": "array", "items": { "type": "string" } }, "title_field": { "title": "Text field to use as document title with Vectara", "description": "A field that will be used to populate the `title` of each document. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered text fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.", "default": "", "always_show": true, "examples": [ "document_key" ], "type": "string" }, "metadata_fields": { "title": "Fields to store as metadata", "description": "List of fields in the record that should be stored as metadata. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered metadata fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. When specifying nested paths, all matching values are flattened into an array set to a field named by the path.", "default": [], "always_show": true, "examples": [ "age", "user" ], "type": "array", "items": { "type": "string" } }, "destinationType": { "title": "vectara", "const": "vectara", "enum": [ "vectara" ], "order": 0, "type": "string" } }, "required": [ "oauth2", "customer_id", "corpus_name", "destinationType" ], "description": "Configuration to connect to the Vectara instance", "groups": [ { "id": "account", "title": "Account" }, { "id": "auth", "title": "Authentication" } ] }, "destination-s3-glue": { "title": "S3 Destination Spec", "type": "object", "required": [ "s3_bucket_name", "s3_bucket_path", "s3_bucket_region", "format", "glue_database", "glue_serialization_library", "destinationType" ], "properties": { "access_key_id": { "type": "string", "description": "The access key ID to access the S3 bucket. Airbyte requires Read and Write permissions to the given bucket. Read more here.", "title": "S3 Key ID", "airbyte_secret": true, "examples": [ "A012345678910EXAMPLE" ], "order": 0, "x-speakeasy-param-sensitive": true }, "secret_access_key": { "type": "string", "description": "The corresponding secret to the access key ID. Read more here", "title": "S3 Access Key", "airbyte_secret": true, "examples": [ "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY" ], "order": 1, "x-speakeasy-param-sensitive": true }, "s3_bucket_name": { "title": "S3 Bucket Name", "type": "string", "description": "The name of the S3 bucket. Read more here.", "examples": [ "airbyte_sync" ], "order": 2 }, "s3_bucket_path": { "title": "S3 Bucket Path", "description": "Directory under the S3 bucket where data will be written. Read more here", "type": "string", "examples": [ "data_sync/test" ], "order": 3 }, "s3_bucket_region": { "title": "S3 Bucket Region", "type": "string", "default": "", "description": "The region of the S3 bucket. See here for all region codes.", "enum": [ "", "af-south-1", "ap-east-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-south-2", "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", "ap-southeast-4", "ca-central-1", "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", "eu-central-2", "eu-north-1", "eu-south-1", "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", "il-central-1", "me-central-1", "me-south-1", "sa-east-1", "us-east-1", "us-east-2", "us-gov-east-1", "us-gov-west-1", "us-west-1", "us-west-2" ], "order": 4 }, "format": { "title": "Output Format", "type": "object", "description": "Format of the data output. See here for more details", "oneOf": [ { "title": "JSON Lines: Newline-delimited JSON", "required": [ "format_type" ], "properties": { "format_type": { "title": "Format Type", "type": "string", "enum": [ "JSONL" ], "default": "JSONL" }, "compression": { "title": "Compression", "type": "object", "description": "Whether the output files should be compressed. If compression is selected, the output filename will have an extra extension (GZIP: \".jsonl.gz\").", "oneOf": [ { "title": "No Compression", "requires": "compression_type", "properties": { "compression_type": { "type": "string", "enum": [ "No Compression" ], "default": "No Compression" } } }, { "title": "GZIP", "requires": "compression_type", "properties": { "compression_type": { "type": "string", "enum": [ "GZIP" ], "default": "GZIP" } } } ] }, "flattening": { "type": "string", "title": "Flattening", "description": "Whether the input json data should be normalized (flattened) in the output JSON Lines. Please refer to docs for details.", "default": "Root level flattening", "enum": [ "No flattening", "Root level flattening" ] } } } ], "order": 5 }, "s3_endpoint": { "title": "Endpoint", "type": "string", "default": "", "description": "Your S3 endpoint url. Read more here", "examples": [ "http://localhost:9000" ], "order": 6 }, "s3_path_format": { "title": "S3 Path Format", "description": "Format string on how data will be organized inside the S3 bucket directory. Read more here", "type": "string", "examples": [ "${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_" ], "order": 7 }, "file_name_pattern": { "type": "string", "description": "The pattern allows you to set the file-name format for the S3 staging file(s)", "title": "S3 Filename pattern", "examples": [ "{date}", "{date:yyyy_MM}", "{timestamp}", "{part_number}", "{sync_id}" ], "order": 8 }, "glue_database": { "type": "string", "description": "Name of the glue database for creating the tables, leave blank if no integration", "title": "Glue database name", "examples": [ "airbyte_database" ], "order": 9 }, "glue_serialization_library": { "title": "Serialization Library", "description": "The library that your query engine will use for reading and writing data in your lake.", "type": "string", "enum": [ "org.openx.data.jsonserde.JsonSerDe", "org.apache.hive.hcatalog.data.JsonSerDe" ], "default": "org.openx.data.jsonserde.JsonSerDe", "order": 10 }, "destinationType": { "title": "s3-glue", "const": "s3-glue", "enum": [ "s3-glue" ], "order": 0, "type": "string" } } }, "destination-dev-null": { "title": "E2E Test (/dev/null) Destination Spec", "type": "object", "required": [ "test_destination", "destinationType" ], "properties": { "test_destination": { "title": "Test Destination", "type": "object", "description": "The type of destination to be used", "oneOf": [ { "title": "Silent", "required": [ "test_destination_type" ], "properties": { "test_destination_type": { "type": "string", "const": "SILENT", "default": "SILENT", "enum": [ "SILENT" ] } } } ] }, "destinationType": { "title": "dev-null", "const": "dev-null", "enum": [ "dev-null" ], "order": 0, "type": "string" } } }, "destination-timeplus": { "title": "Destination Timeplus", "type": "object", "required": [ "endpoint", "apikey", "destinationType" ], "properties": { "endpoint": { "title": "Endpoint", "description": "Timeplus workspace endpoint", "type": "string", "default": "https://us.timeplus.cloud/", "examples": [ "https://us.timeplus.cloud/workspace_id" ], "order": 0 }, "apikey": { "title": "API key", "description": "Personal API key", "type": "string", "airbyte_secret": true, "order": 1, "x-speakeasy-param-sensitive": true }, "destinationType": { "title": "timeplus", "const": "timeplus", "enum": [ "timeplus" ], "order": 0, "type": "string" } } }, "destination-convex": { "title": "Destination Convex", "type": "object", "required": [ "deployment_url", "access_key", "destinationType" ], "properties": { "deployment_url": { "type": "string", "description": "URL of the Convex deployment that is the destination", "examples": [ "https://murky-swan-635.convex.cloud", "https://cluttered-owl-337.convex.cloud" ] }, "access_key": { "type": "string", "description": "API access key used to send data to a Convex deployment.", "airbyte_secret": "true", "x-speakeasy-param-sensitive": true }, "destinationType": { "title": "convex", "const": "convex", "enum": [ "convex" ], "order": 0, "type": "string" } } }, "destination-firestore": { "title": "Destination Google Firestore", "type": "object", "required": [ "project_id", "destinationType" ], "properties": { "project_id": { "type": "string", "description": "The GCP project ID for the project containing the target BigQuery dataset.", "title": "Project ID" }, "credentials_json": { "type": "string", "description": "The contents of the JSON service account key. Check out the docs if you need help generating this key. Default credentials will be used if this field is left empty.", "title": "Credentials JSON", "airbyte_secret": true }, "destinationType": { "title": "firestore", "const": "firestore", "enum": [ "firestore" ], "order": 0, "type": "string" } } }, "destination-redshift": { "title": "Redshift Destination Spec", "type": "object", "required": [ "host", "port", "database", "username", "password", "schema", "destinationType" ], "properties": { "host": { "description": "Host Endpoint of the Redshift Cluster (must include the cluster-id, region and end with .redshift.amazonaws.com)", "type": "string", "title": "Host", "group": "connection", "order": 1 }, "port": { "description": "Port of the database.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 5439, "examples": [ "5439" ], "title": "Port", "group": "connection", "order": 2 }, "username": { "description": "Username to use to access the database.", "type": "string", "title": "Username", "group": "connection", "order": 3 }, "password": { "description": "Password associated with the username.", "type": "string", "airbyte_secret": true, "title": "Password", "group": "connection", "order": 4, "x-speakeasy-param-sensitive": true }, "database": { "description": "Name of the database.", "type": "string", "title": "Database", "group": "connection", "order": 5 }, "schema": { "description": "The default schema tables are written to if the source does not specify a namespace. Unless specifically configured, the usual value for this field is \"public\".", "type": "string", "examples": [ "public" ], "default": "public", "group": "connection", "title": "Default Schema", "order": 6 }, "jdbc_url_params": { "title": "JDBC URL Params", "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "type": "string", "group": "connection", "order": 7 }, "uploading_method": { "title": "Uploading Method", "type": "object", "description": "The way data will be uploaded to Redshift.", "group": "connection", "order": 8, "display_type": "radio", "oneOf": [ { "title": "AWS S3 Staging", "description": "(recommended) Uploads data to S3 and then uses a COPY to insert the data into Redshift. COPY is recommended for production workloads for better speed and scalability. See AWS docs for more details.", "required": [ "method", "s3_bucket_name", "s3_bucket_region", "access_key_id", "secret_access_key" ], "properties": { "method": { "type": "string", "const": "S3 Staging", "enum": [ "S3 Staging" ] }, "s3_bucket_name": { "title": "S3 Bucket Name", "type": "string", "description": "The name of the staging S3 bucket.", "examples": [ "airbyte.staging" ], "order": 0 }, "s3_bucket_path": { "title": "S3 Bucket Path", "type": "string", "description": "The directory under the S3 bucket where data will be written. If not provided, then defaults to the root directory. See path's name recommendations for more details.", "examples": [ "data_sync/test" ], "order": 1 }, "s3_bucket_region": { "title": "S3 Bucket Region", "type": "string", "default": "", "description": "The region of the S3 staging bucket.", "enum": [ "", "af-south-1", "ap-east-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-south-2", "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", "ap-southeast-4", "ca-central-1", "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", "eu-central-2", "eu-north-1", "eu-south-1", "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", "il-central-1", "me-central-1", "me-south-1", "sa-east-1", "sa-east-1", "us-east-1", "us-east-2", "us-gov-east-1", "us-gov-west-1", "us-west-1", "us-west-2" ], "order": 2 }, "access_key_id": { "type": "string", "description": "This ID grants access to the above S3 staging bucket. Airbyte requires Read and Write permissions to the given bucket. See AWS docs on how to generate an access key ID and secret access key.", "title": "S3 Access Key Id", "airbyte_secret": true, "order": 3, "x-speakeasy-param-sensitive": true }, "secret_access_key": { "type": "string", "description": "The corresponding secret to the above access key id. See AWS docs on how to generate an access key ID and secret access key.", "title": "S3 Secret Access Key", "airbyte_secret": true, "order": 4, "x-speakeasy-param-sensitive": true }, "file_name_pattern": { "type": "string", "description": "The pattern allows you to set the file-name format for the S3 staging file(s)", "title": "S3 Filename pattern", "examples": [ "{date}", "{date:yyyy_MM}", "{timestamp}", "{part_number}", "{sync_id}" ], "order": 5 }, "purge_staging_data": { "title": "Purge Staging Files and Tables", "type": "boolean", "description": "Whether to delete the staging files from S3 after completing the sync. See docs for details.", "default": true, "order": 6 }, "encryption": { "title": "Encryption", "type": "object", "description": "How to encrypt the staging data", "default": { "encryption_type": "none" }, "oneOf": [ { "title": "No encryption", "description": "Staging data will be stored in plaintext.", "type": "object", "required": [ "encryption_type" ], "properties": { "encryption_type": { "type": "string", "const": "none", "enum": [ "none" ], "default": "none" } } }, { "title": "AES-CBC envelope encryption", "description": "Staging data will be encrypted using AES-CBC envelope encryption.", "type": "object", "required": [ "encryption_type" ], "properties": { "encryption_type": { "type": "string", "const": "aes_cbc_envelope", "enum": [ "aes_cbc_envelope" ], "default": "aes_cbc_envelope" }, "key_encrypting_key": { "type": "string", "title": "Key", "description": "The key, base64-encoded. Must be either 128, 192, or 256 bits. Leave blank to have Airbyte generate an ephemeral key for each sync.", "airbyte_secret": true, "x-speakeasy-param-sensitive": true } } } ], "order": 7 }, "file_buffer_count": { "title": "File Buffer Count", "type": "integer", "minimum": 10, "maximum": 50, "default": 10, "description": "Number of file buffers allocated for writing data. Increasing this number is beneficial for connections using Change Data Capture (CDC) and up to the number of streams within a connection. Increasing the number of file buffers past the maximum number of streams has deteriorating effects", "examples": [ "10" ] } } }, { "title": "Standard", "required": [ "method" ], "description": "(not recommended) Direct loading using SQL INSERT statements. This method is extremely inefficient and provided only for quick testing. In all other cases, you should use S3 uploading.", "properties": { "method": { "type": "string", "const": "Standard", "enum": [ "Standard" ] } } } ] }, "raw_data_schema": { "type": "string", "description": "The schema to write raw tables into", "title": "Destinations V2 Raw Table Schema", "order": 9, "group": "connection" }, "enable_incremental_final_table_updates": { "type": "boolean", "default": false, "description": "When enabled your data will load into your final tables incrementally while your data is still being synced. When Disabled (the default), your data loads into your final tables once at the end of a sync. Note that this option only applies if you elect to create Final tables", "title": "Enable Loading Data Incrementally to Final Tables", "order": 10, "group": "connection" }, "disable_type_dedupe": { "type": "boolean", "default": false, "description": "Disable Writing Final Tables. WARNING! The data format in _airbyte_data is likely stable but there are no guarantees that other metadata columns will remain the same in future versions", "title": "Disable Final Tables. (WARNING! Unstable option; Columns in raw table schema might change between versions)", "order": 11, "group": "connection" }, "tunnel_method": { "type": "object", "title": "SSH Tunnel Method", "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", "oneOf": [ { "title": "No Tunnel", "required": [ "tunnel_method" ], "properties": { "tunnel_method": { "description": "No ssh tunnel needed to connect to database", "type": "string", "const": "NO_TUNNEL", "order": 0, "enum": [ "NO_TUNNEL" ] } } }, { "title": "SSH Key Authentication", "required": [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "ssh_key" ], "properties": { "tunnel_method": { "description": "Connect through a jump server tunnel host using username and ssh key", "type": "string", "const": "SSH_KEY_AUTH", "order": 0, "enum": [ "SSH_KEY_AUTH" ] }, "tunnel_host": { "title": "SSH Tunnel Jump Server Host", "description": "Hostname of the jump server host that allows inbound ssh tunnel.", "type": "string", "order": 1 }, "tunnel_port": { "title": "SSH Connection Port", "description": "Port on the proxy/jump server that accepts inbound ssh connections.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 22, "examples": [ "22" ], "order": 2 }, "tunnel_user": { "title": "SSH Login Username", "description": "OS-level username for logging into the jump server host.", "type": "string", "order": 3 }, "ssh_key": { "title": "SSH Private Key", "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", "type": "string", "airbyte_secret": true, "multiline": true, "order": 4, "x-speakeasy-param-sensitive": true } } }, { "title": "Password Authentication", "required": [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "tunnel_user_password" ], "properties": { "tunnel_method": { "description": "Connect through a jump server tunnel host using username and password authentication", "type": "string", "const": "SSH_PASSWORD_AUTH", "order": 0, "enum": [ "SSH_PASSWORD_AUTH" ] }, "tunnel_host": { "title": "SSH Tunnel Jump Server Host", "description": "Hostname of the jump server host that allows inbound ssh tunnel.", "type": "string", "order": 1 }, "tunnel_port": { "title": "SSH Connection Port", "description": "Port on the proxy/jump server that accepts inbound ssh connections.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 22, "examples": [ "22" ], "order": 2 }, "tunnel_user": { "title": "SSH Login Username", "description": "OS-level username for logging into the jump server host", "type": "string", "order": 3 }, "tunnel_user_password": { "title": "Password", "description": "OS-level password for logging into the jump server host", "type": "string", "airbyte_secret": true, "order": 4, "x-speakeasy-param-sensitive": true } } } ] }, "destinationType": { "title": "redshift", "const": "redshift", "enum": [ "redshift" ], "order": 0, "type": "string" } }, "groups": [ { "id": "connection", "title": "Connection" } ] }, "destination-dynamodb": { "title": "DynamoDB Destination Spec", "type": "object", "required": [ "dynamodb_table_name_prefix", "dynamodb_region", "access_key_id", "secret_access_key", "destinationType" ], "properties": { "dynamodb_endpoint": { "title": "Endpoint", "type": "string", "default": "", "description": "This is your DynamoDB endpoint url.(if you are working with AWS DynamoDB, just leave empty).", "examples": [ "http://localhost:9000" ] }, "dynamodb_table_name_prefix": { "title": "Table name prefix", "type": "string", "description": "The prefix to use when naming DynamoDB tables.", "examples": [ "airbyte_sync" ] }, "dynamodb_region": { "title": "DynamoDB Region", "type": "string", "default": "", "description": "The region of the DynamoDB.", "enum": [ "", "af-south-1", "ap-east-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-south-2", "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", "ap-southeast-4", "ca-central-1", "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", "eu-central-2", "eu-north-1", "eu-south-1", "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", "il-central-1", "me-central-1", "me-south-1", "sa-east-1", "us-east-1", "us-east-2", "us-gov-east-1", "us-gov-west-1", "us-west-1", "us-west-2" ] }, "access_key_id": { "type": "string", "description": "The access key id to access the DynamoDB. Airbyte requires Read and Write permissions to the DynamoDB.", "title": "DynamoDB Key Id", "airbyte_secret": true, "examples": [ "A012345678910EXAMPLE" ], "x-speakeasy-param-sensitive": true }, "secret_access_key": { "type": "string", "description": "The corresponding secret to the access key id.", "title": "DynamoDB Access Key", "airbyte_secret": true, "examples": [ "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY" ], "x-speakeasy-param-sensitive": true }, "destinationType": { "title": "dynamodb", "const": "dynamodb", "enum": [ "dynamodb" ], "order": 0, "type": "string" } } }, "destination-qdrant": { "title": "Destination Config", "description": "The configuration model for the Vector DB based destinations. This model is used to generate the UI for the destination configuration,\nas well as to provide type safety for the configuration passed to the destination.\n\nThe configuration model is composed of four parts:\n* Processing configuration\n* Embedding configuration\n* Indexing configuration\n* Advanced configuration\n\nProcessing, embedding and advanced configuration are provided by this base class, while the indexing configuration is provided by the destination connector in the sub class.", "type": "object", "properties": { "embedding": { "title": "Embedding", "description": "Embedding configuration", "group": "embedding", "type": "object", "oneOf": [ { "title": "OpenAI", "type": "object", "properties": { "mode": { "title": "Mode", "default": "openai", "const": "openai", "enum": [ "openai" ], "type": "string" }, "openai_key": { "title": "OpenAI API key", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true } }, "required": [ "openai_key", "mode" ], "description": "Use the OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." }, { "title": "Cohere", "type": "object", "properties": { "mode": { "title": "Mode", "default": "cohere", "const": "cohere", "enum": [ "cohere" ], "type": "string" }, "cohere_key": { "title": "Cohere API key", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true } }, "required": [ "cohere_key", "mode" ], "description": "Use the Cohere API to embed text." }, { "title": "Fake", "type": "object", "properties": { "mode": { "title": "Mode", "default": "fake", "const": "fake", "enum": [ "fake" ], "type": "string" } }, "description": "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs.", "required": [ "mode" ] }, { "title": "Azure OpenAI", "type": "object", "properties": { "mode": { "title": "Mode", "default": "azure_openai", "const": "azure_openai", "enum": [ "azure_openai" ], "type": "string" }, "openai_key": { "title": "Azure OpenAI API key", "description": "The API key for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true }, "api_base": { "title": "Resource base URL", "description": "The base URL for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", "examples": [ "https://your-resource-name.openai.azure.com" ], "type": "string" }, "deployment": { "title": "Deployment", "description": "The deployment for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", "examples": [ "your-resource-name" ], "type": "string" } }, "required": [ "openai_key", "api_base", "deployment", "mode" ], "description": "Use the Azure-hosted OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." }, { "title": "OpenAI-compatible", "type": "object", "properties": { "mode": { "title": "Mode", "default": "openai_compatible", "const": "openai_compatible", "enum": [ "openai_compatible" ], "type": "string" }, "api_key": { "title": "API key", "default": "", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true }, "base_url": { "title": "Base URL", "description": "The base URL for your OpenAI-compatible service", "examples": [ "https://your-service-name.com" ], "type": "string" }, "model_name": { "title": "Model name", "description": "The name of the model to use for embedding", "default": "text-embedding-ada-002", "examples": [ "text-embedding-ada-002" ], "type": "string" }, "dimensions": { "title": "Embedding dimensions", "description": "The number of dimensions the embedding model is generating", "examples": [ 1536, 384 ], "type": "integer" } }, "required": [ "base_url", "dimensions", "mode" ], "description": "Use a service that's compatible with the OpenAI API to embed text." } ] }, "processing": { "title": "ProcessingConfigModel", "type": "object", "properties": { "chunk_size": { "title": "Chunk size", "description": "Size of chunks in tokens to store in vector store (make sure it is not too big for the context if your LLM)", "maximum": 8191, "minimum": 1, "type": "integer" }, "chunk_overlap": { "title": "Chunk overlap", "description": "Size of overlap between chunks in tokens to store in vector store to better capture relevant context", "default": 0, "type": "integer" }, "text_fields": { "title": "Text fields to embed", "description": "List of fields in the record that should be used to calculate the embedding. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered text fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.", "default": [], "always_show": true, "examples": [ "text", "user.name", "users.*.name" ], "type": "array", "items": { "type": "string" } }, "metadata_fields": { "title": "Fields to store as metadata", "description": "List of fields in the record that should be stored as metadata. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered metadata fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. When specifying nested paths, all matching values are flattened into an array set to a field named by the path.", "default": [], "always_show": true, "examples": [ "age", "user", "user.name" ], "type": "array", "items": { "type": "string" } }, "text_splitter": { "title": "Text splitter", "description": "Split text fields into chunks based on the specified method.", "type": "object", "oneOf": [ { "title": "By Separator", "type": "object", "properties": { "mode": { "title": "Mode", "default": "separator", "const": "separator", "enum": [ "separator" ], "type": "string" }, "separators": { "title": "Separators", "description": "List of separator strings to split text fields by. The separator itself needs to be wrapped in double quotes, e.g. to split by the dot character, use \".\". To split by a newline, use \"\\n\".", "default": [ "\"\\n\\n\"", "\"\\n\"", "\" \"", "\"\"" ], "type": "array", "items": { "type": "string" } }, "keep_separator": { "title": "Keep separator", "description": "Whether to keep the separator in the resulting chunks", "default": false, "type": "boolean" } }, "description": "Split the text by the list of separators until the chunk size is reached, using the earlier mentioned separators where possible. This is useful for splitting text fields by paragraphs, sentences, words, etc.", "required": [ "mode" ] }, { "title": "By Markdown header", "type": "object", "properties": { "mode": { "title": "Mode", "default": "markdown", "const": "markdown", "enum": [ "markdown" ], "type": "string" }, "split_level": { "title": "Split level", "description": "Level of markdown headers to split text fields by. Headings down to the specified level will be used as split points", "default": 1, "minimum": 1, "maximum": 6, "type": "integer" } }, "description": "Split the text by Markdown headers down to the specified header level. If the chunk size fits multiple sections, they will be combined into a single chunk.", "required": [ "mode" ] }, { "title": "By Programming Language", "type": "object", "properties": { "mode": { "title": "Mode", "default": "code", "const": "code", "enum": [ "code" ], "type": "string" }, "language": { "title": "Language", "description": "Split code in suitable places based on the programming language", "enum": [ "cpp", "go", "java", "js", "php", "proto", "python", "rst", "ruby", "rust", "scala", "swift", "markdown", "latex", "html", "sol" ], "type": "string" } }, "required": [ "language", "mode" ], "description": "Split the text by suitable delimiters based on the programming language. This is useful for splitting code into chunks." } ] }, "field_name_mappings": { "title": "Field name mappings", "description": "List of fields to rename. Not applicable for nested fields, but can be used to rename fields already flattened via dot notation.", "default": [], "type": "array", "items": { "title": "FieldNameMappingConfigModel", "type": "object", "properties": { "from_field": { "title": "From field name", "description": "The field name in the source", "type": "string" }, "to_field": { "title": "To field name", "description": "The field name to use in the destination", "type": "string" } }, "required": [ "from_field", "to_field" ] } } }, "required": [ "chunk_size" ], "group": "processing" }, "omit_raw_text": { "title": "Do not store raw text", "description": "Do not store the text that gets embedded along with the vector and the metadata in the destination. If set to true, only the vector and the metadata will be stored - in this case raw text for LLM use cases needs to be retrieved from another source.", "default": false, "group": "advanced", "type": "boolean" }, "indexing": { "title": "Indexing", "type": "object", "properties": { "url": { "title": "Public Endpoint", "description": "Public Endpoint of the Qdrant cluser", "order": 0, "type": "string" }, "auth_method": { "title": "Authentication Method", "description": "Method to authenticate with the Qdrant Instance", "default": "api_key_auth", "type": "object", "order": 1, "oneOf": [ { "title": "ApiKeyAuth", "type": "object", "properties": { "mode": { "title": "Mode", "default": "api_key_auth", "const": "api_key_auth", "enum": [ "api_key_auth" ], "type": "string" }, "api_key": { "title": "API Key", "description": "API Key for the Qdrant instance", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true } }, "required": [ "api_key" ] }, { "title": "NoAuth", "type": "object", "properties": { "mode": { "title": "Mode", "default": "no_auth", "const": "no_auth", "enum": [ "no_auth" ], "type": "string" } } } ] }, "prefer_grpc": { "title": "Prefer gRPC", "description": "Whether to prefer gRPC over HTTP. Set to true for Qdrant cloud clusters", "default": true, "type": "boolean" }, "collection": { "title": "Collection Name", "description": "The collection to load data into", "order": 2, "type": "string" }, "distance_metric": { "title": "Distance Metric", "description": "The Distance metric used to measure similarities among vectors. This field is only used if the collection defined in the does not exist yet and is created automatically by the connector.", "default": "cos", "enum": [ "dot", "cos", "euc" ], "type": "string" }, "text_field": { "title": "Text Field", "description": "The field in the payload that contains the embedded text", "default": "text", "type": "string" } }, "required": [ "url", "collection" ], "group": "Indexing", "description": "Indexing configuration" }, "destinationType": { "title": "qdrant", "const": "qdrant", "enum": [ "qdrant" ], "order": 0, "type": "string" } }, "required": [ "embedding", "processing", "indexing", "destinationType" ], "groups": [ { "id": "processing", "title": "Processing" }, { "id": "embedding", "title": "Embedding" }, { "id": "indexing", "title": "Indexing" }, { "id": "advanced", "title": "Advanced" } ] }, "destination-snowflake": { "title": "Snowflake Destination Spec", "type": "object", "required": [ "host", "role", "warehouse", "database", "schema", "username", "destinationType" ], "properties": { "host": { "description": "Enter your Snowflake account's locator (in the format ...snowflakecomputing.com)", "examples": [ "accountname.us-east-2.aws.snowflakecomputing.com", "accountname.snowflakecomputing.com" ], "type": "string", "title": "Host", "pattern": "^(http(s)?:\\/\\/)?([^./?#]+\\.)?([^./?#]+\\.)?([^./?#]+\\.)?([^./?#]+\\.snowflakecomputing\\.com)$", "pattern_descriptor": "{account_name}.snowflakecomputing.com or {accountname}.{aws_location}.aws.snowflakecomputing.com", "order": 0 }, "role": { "description": "Enter the role that you want to use to access Snowflake", "examples": [ "AIRBYTE_ROLE" ], "type": "string", "title": "Role", "order": 1 }, "warehouse": { "description": "Enter the name of the warehouse that you want to sync data into", "examples": [ "AIRBYTE_WAREHOUSE" ], "type": "string", "title": "Warehouse", "order": 2 }, "database": { "description": "Enter the name of the database you want to sync data into", "examples": [ "AIRBYTE_DATABASE" ], "type": "string", "title": "Database", "order": 3 }, "schema": { "description": "Enter the name of the default schema", "examples": [ "AIRBYTE_SCHEMA" ], "type": "string", "title": "Default Schema", "order": 4 }, "username": { "description": "Enter the name of the user you want to use to access the database", "examples": [ "AIRBYTE_USER" ], "type": "string", "title": "Username", "order": 5 }, "credentials": { "title": "Authorization Method", "description": "", "type": "object", "oneOf": [ { "title": "Key Pair Authentication", "type": "object", "order": 0, "required": [ "private_key" ], "properties": { "auth_type": { "type": "string", "const": "Key Pair Authentication", "enum": [ "Key Pair Authentication" ], "default": "Key Pair Authentication", "order": 0 }, "private_key": { "type": "string", "title": "Private Key", "description": "RSA Private key to use for Snowflake connection. See the docs for more information on how to obtain this key.", "multiline": true, "airbyte_secret": true, "x-speakeasy-param-sensitive": true }, "private_key_password": { "type": "string", "title": "Passphrase", "description": "Passphrase for private key", "airbyte_secret": true, "x-speakeasy-param-sensitive": true } } }, { "title": "Username and Password", "type": "object", "required": [ "password" ], "order": 1, "properties": { "auth_type": { "type": "string", "const": "Username and Password", "enum": [ "Username and Password" ], "default": "Username and Password", "order": 0 }, "password": { "description": "Enter the password associated with the username.", "type": "string", "airbyte_secret": true, "title": "Password", "order": 1, "x-speakeasy-param-sensitive": true } } }, { "title": "OAuth2.0", "type": "object", "order": 2, "required": [ "access_token", "refresh_token" ], "properties": { "auth_type": { "type": "string", "const": "OAuth2.0", "enum": [ "OAuth2.0" ], "default": "OAuth2.0", "order": 0 }, "client_id": { "type": "string", "title": "Client ID", "description": "Enter your application's Client ID", "airbyte_secret": true }, "client_secret": { "type": "string", "title": "Client Secret", "description": "Enter your application's Client secret", "airbyte_secret": true }, "access_token": { "type": "string", "title": "Access Token", "description": "Enter you application's Access Token", "airbyte_secret": true, "x-speakeasy-param-sensitive": true }, "refresh_token": { "type": "string", "title": "Refresh Token", "description": "Enter your application's Refresh Token", "airbyte_secret": true, "x-speakeasy-param-sensitive": true } } } ], "order": 6 }, "jdbc_url_params": { "description": "Enter the additional properties to pass to the JDBC URL string when connecting to the database (formatted as key=value pairs separated by the symbol &). Example: key1=value1&key2=value2&key3=value3", "title": "JDBC URL Params", "type": "string", "order": 7 }, "raw_data_schema": { "type": "string", "description": "The schema to write raw tables into (default: airbyte_internal)", "title": "Raw Table Schema Name", "order": 10 }, "disable_type_dedupe": { "type": "boolean", "default": false, "description": "Disable Writing Final Tables. WARNING! The data format in _airbyte_data is likely stable but there are no guarantees that other metadata columns will remain the same in future versions", "title": "Disable Final Tables. (WARNING! Unstable option; Columns in raw table schema might change between versions)", "order": 11 }, "enable_incremental_final_table_updates": { "type": "boolean", "default": false, "description": "When enabled your data will load into your final tables incrementally while your data is still being synced. When Disabled (the default), your data loads into your final tables once at the end of a sync. Note that this option only applies if you elect to create Final tables", "title": "Enable Loading Data Incrementally to Final Tables", "order": 12 }, "destinationType": { "title": "snowflake", "const": "snowflake", "enum": [ "snowflake" ], "order": 0, "type": "string" } } }, "destination-databricks": { "title": "Databricks Lakehouse Destination Spec", "type": "object", "required": [ "accept_terms", "databricks_server_hostname", "databricks_http_path", "databricks_personal_access_token", "data_source", "destinationType" ], "properties": { "accept_terms": { "title": "Agree to the Databricks JDBC Driver Terms & Conditions", "type": "boolean", "description": "You must agree to the Databricks JDBC Driver Terms & Conditions to use this connector.", "default": false, "order": 1 }, "databricks_server_hostname": { "title": "Server Hostname", "type": "string", "description": "Databricks Cluster Server Hostname.", "examples": [ "abc-12345678-wxyz.cloud.databricks.com" ], "order": 2 }, "databricks_http_path": { "title": "HTTP Path", "type": "string", "description": "Databricks Cluster HTTP Path.", "examples": [ "sql/protocolvx/o/1234567489/0000-1111111-abcd90" ], "order": 3 }, "databricks_port": { "title": "Port", "type": "string", "description": "Databricks Cluster Port.", "default": "443", "examples": [ "443" ], "order": 4 }, "databricks_personal_access_token": { "title": "Access Token", "type": "string", "description": "Databricks Personal Access Token for making authenticated requests.", "examples": [ "dapi0123456789abcdefghij0123456789AB" ], "airbyte_secret": true, "order": 5, "x-speakeasy-param-sensitive": true }, "database": { "title": "Databricks catalog", "description": "The name of the catalog. If not specified otherwise, the \"hive_metastore\" will be used.", "type": "string", "order": 6 }, "schema": { "title": "Default Schema", "description": "The default schema tables are written. If not specified otherwise, the \"default\" will be used.", "type": "string", "examples": [ "default" ], "default": "default", "order": 7 }, "enable_schema_evolution": { "title": "Support schema evolution for all streams.", "type": "boolean", "description": "Support schema evolution for all streams. If \"false\", the connector might fail when a stream's schema changes.", "default": false, "order": 8 }, "data_source": { "title": "Data Source", "type": "object", "description": "Storage on which the delta lake is built.", "default": "MANAGED_TABLES_STORAGE", "order": 9, "oneOf": [ { "title": "[Recommended] Managed tables", "required": [ "data_source_type" ], "properties": { "data_source_type": { "type": "string", "const": "MANAGED_TABLES_STORAGE", "order": 0, "enum": [ "MANAGED_TABLES_STORAGE" ] } } }, { "title": "Amazon S3", "required": [ "data_source_type", "s3_bucket_name", "s3_bucket_path", "s3_bucket_region", "s3_access_key_id", "s3_secret_access_key" ], "properties": { "data_source_type": { "type": "string", "const": "S3_STORAGE", "order": 1, "enum": [ "S3_STORAGE" ] }, "s3_bucket_name": { "title": "S3 Bucket Name", "type": "string", "description": "The name of the S3 bucket to use for intermittent staging of the data.", "examples": [ "airbyte.staging" ], "order": 2 }, "s3_bucket_path": { "title": "S3 Bucket Path", "type": "string", "description": "The directory under the S3 bucket where data will be written.", "examples": [ "data_sync/test" ], "order": 3 }, "s3_bucket_region": { "title": "S3 Bucket Region", "type": "string", "default": "", "description": "The region of the S3 staging bucket to use if utilising a copy strategy.", "enum": [ "", "us-east-1", "us-east-2", "us-west-1", "us-west-2", "af-south-1", "ap-east-1", "ap-south-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-southeast-1", "ap-southeast-2", "ca-central-1", "cn-north-1", "cn-northwest-1", "eu-central-1", "eu-north-1", "eu-south-1", "eu-west-1", "eu-west-2", "eu-west-3", "sa-east-1", "me-south-1", "us-gov-east-1", "us-gov-west-1" ], "order": 4 }, "s3_access_key_id": { "type": "string", "description": "The Access Key Id granting allow one to access the above S3 staging bucket. Airbyte requires Read and Write permissions to the given bucket.", "title": "S3 Access Key ID", "examples": [ "A012345678910EXAMPLE" ], "airbyte_secret": true, "order": 5, "x-speakeasy-param-sensitive": true }, "s3_secret_access_key": { "title": "S3 Secret Access Key", "type": "string", "description": "The corresponding secret to the above access key id.", "examples": [ "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY" ], "airbyte_secret": true, "order": 6, "x-speakeasy-param-sensitive": true }, "file_name_pattern": { "type": "string", "description": "The pattern allows you to set the file-name format for the S3 staging file(s)", "title": "S3 Filename pattern", "examples": [ "{date}", "{date:yyyy_MM}", "{timestamp}", "{part_number}", "{sync_id}" ], "order": 7 } } }, { "title": "Azure Blob Storage", "required": [ "data_source_type", "azure_blob_storage_account_name", "azure_blob_storage_container_name", "azure_blob_storage_sas_token" ], "properties": { "data_source_type": { "type": "string", "const": "AZURE_BLOB_STORAGE", "order": 0, "enum": [ "AZURE_BLOB_STORAGE" ] }, "azure_blob_storage_endpoint_domain_name": { "title": "Endpoint Domain Name", "type": "string", "default": "blob.core.windows.net", "description": "This is Azure Blob Storage endpoint domain name. Leave default value (or leave it empty if run container from command line) to use Microsoft native from example.", "examples": [ "blob.core.windows.net" ], "order": 1 }, "azure_blob_storage_account_name": { "title": "Azure Blob Storage Account Name", "type": "string", "description": "The account's name of the Azure Blob Storage.", "examples": [ "airbyte5storage" ], "order": 2 }, "azure_blob_storage_container_name": { "title": "Azure Blob Storage Container Name", "type": "string", "description": "The name of the Azure blob storage container.", "examples": [ "airbytetestcontainername" ], "order": 3 }, "azure_blob_storage_sas_token": { "title": "SAS Token", "type": "string", "airbyte_secret": true, "description": "Shared access signature (SAS) token to grant limited access to objects in your storage account.", "examples": [ "?sv=2016-05-31&ss=b&srt=sco&sp=rwdl&se=2018-06-27T10:05:50Z&st=2017-06-27T02:05:50Z&spr=https,http&sig=bgqQwoXwxzuD2GJfagRg7VOS8hzNr3QLT7rhS8OFRLQ%3D" ], "order": 4, "x-speakeasy-param-sensitive": true } } } ] }, "purge_staging_data": { "title": "Purge Staging Files and Tables", "type": "boolean", "description": "Default to 'true'. Switch it to 'false' for debugging purpose.", "default": true, "order": 10 }, "destinationType": { "title": "databricks", "const": "databricks", "enum": [ "databricks" ], "order": 0, "type": "string" } } }, "destination-oracle": { "title": "Oracle Destination Spec", "type": "object", "required": [ "host", "port", "username", "sid", "destinationType" ], "properties": { "host": { "title": "Host", "description": "The hostname of the database.", "type": "string", "order": 0 }, "port": { "title": "Port", "description": "The port of the database.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 1521, "examples": [ "1521" ], "order": 1 }, "sid": { "title": "SID", "description": "The System Identifier uniquely distinguishes the instance from any other instance on the same computer.", "type": "string", "order": 2 }, "username": { "title": "User", "description": "The username to access the database. This user must have CREATE USER privileges in the database.", "type": "string", "order": 3 }, "password": { "title": "Password", "description": "The password associated with the username.", "type": "string", "airbyte_secret": true, "order": 4, "x-speakeasy-param-sensitive": true }, "jdbc_url_params": { "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "title": "JDBC URL Params", "type": "string", "order": 5 }, "schema": { "title": "Default Schema", "description": "The default schema is used as the target schema for all statements issued from the connection that do not explicitly specify a schema name. The usual value for this field is \"airbyte\". In Oracle, schemas and users are the same thing, so the \"user\" parameter is used as the login credentials and this is used for the default Airbyte message schema.", "type": "string", "examples": [ "airbyte" ], "default": "airbyte", "order": 6 }, "tunnel_method": { "type": "object", "title": "SSH Tunnel Method", "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", "oneOf": [ { "title": "No Tunnel", "required": [ "tunnel_method" ], "properties": { "tunnel_method": { "description": "No ssh tunnel needed to connect to database", "type": "string", "const": "NO_TUNNEL", "order": 0, "enum": [ "NO_TUNNEL" ] } } }, { "title": "SSH Key Authentication", "required": [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "ssh_key" ], "properties": { "tunnel_method": { "description": "Connect through a jump server tunnel host using username and ssh key", "type": "string", "const": "SSH_KEY_AUTH", "order": 0, "enum": [ "SSH_KEY_AUTH" ] }, "tunnel_host": { "title": "SSH Tunnel Jump Server Host", "description": "Hostname of the jump server host that allows inbound ssh tunnel.", "type": "string", "order": 1 }, "tunnel_port": { "title": "SSH Connection Port", "description": "Port on the proxy/jump server that accepts inbound ssh connections.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 22, "examples": [ "22" ], "order": 2 }, "tunnel_user": { "title": "SSH Login Username", "description": "OS-level username for logging into the jump server host.", "type": "string", "order": 3 }, "ssh_key": { "title": "SSH Private Key", "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", "type": "string", "airbyte_secret": true, "multiline": true, "order": 4, "x-speakeasy-param-sensitive": true } } }, { "title": "Password Authentication", "required": [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "tunnel_user_password" ], "properties": { "tunnel_method": { "description": "Connect through a jump server tunnel host using username and password authentication", "type": "string", "const": "SSH_PASSWORD_AUTH", "order": 0, "enum": [ "SSH_PASSWORD_AUTH" ] }, "tunnel_host": { "title": "SSH Tunnel Jump Server Host", "description": "Hostname of the jump server host that allows inbound ssh tunnel.", "type": "string", "order": 1 }, "tunnel_port": { "title": "SSH Connection Port", "description": "Port on the proxy/jump server that accepts inbound ssh connections.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 22, "examples": [ "22" ], "order": 2 }, "tunnel_user": { "title": "SSH Login Username", "description": "OS-level username for logging into the jump server host", "type": "string", "order": 3 }, "tunnel_user_password": { "title": "Password", "description": "OS-level password for logging into the jump server host", "type": "string", "airbyte_secret": true, "order": 4, "x-speakeasy-param-sensitive": true } } } ] }, "destinationType": { "title": "oracle", "const": "oracle", "enum": [ "oracle" ], "order": 0, "type": "string" } } }, "destination-aws-datalake": { "title": "AWS Datalake Destination Spec", "type": "object", "required": [ "credentials", "region", "bucket_name", "lakeformation_database_name", "destinationType" ], "properties": { "aws_account_id": { "type": "string", "title": "AWS Account Id", "description": "target aws account id", "examples": [ "111111111111" ], "order": 1 }, "credentials": { "title": "Authentication mode", "description": "Choose How to Authenticate to AWS.", "type": "object", "oneOf": [ { "type": "object", "title": "IAM Role", "required": [ "role_arn", "credentials_title" ], "properties": { "credentials_title": { "type": "string", "title": "Credentials Title", "description": "Name of the credentials", "const": "IAM Role", "enum": [ "IAM Role" ], "default": "IAM Role", "order": 0 }, "role_arn": { "title": "Target Role Arn", "type": "string", "description": "Will assume this role to write data to s3", "airbyte_secret": false } } }, { "type": "object", "title": "IAM User", "required": [ "credentials_title", "aws_access_key_id", "aws_secret_access_key" ], "properties": { "credentials_title": { "type": "string", "title": "Credentials Title", "description": "Name of the credentials", "const": "IAM User", "enum": [ "IAM User" ], "default": "IAM User", "order": 0 }, "aws_access_key_id": { "title": "Access Key Id", "type": "string", "description": "AWS User Access Key Id", "airbyte_secret": true, "x-speakeasy-param-sensitive": true }, "aws_secret_access_key": { "title": "Secret Access Key", "type": "string", "description": "Secret Access Key", "airbyte_secret": true, "x-speakeasy-param-sensitive": true } } } ], "order": 2 }, "region": { "title": "S3 Bucket Region", "type": "string", "default": "", "description": "The region of the S3 bucket. See here for all region codes.", "enum": [ "", "af-south-1", "ap-east-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-south-2", "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", "ap-southeast-4", "ca-central-1", "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", "eu-central-2", "eu-north-1", "eu-south-1", "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", "il-central-1", "me-central-1", "me-south-1", "sa-east-1", "us-east-1", "us-east-2", "us-gov-east-1", "us-gov-west-1", "us-west-1", "us-west-2" ], "order": 3 }, "bucket_name": { "title": "S3 Bucket Name", "type": "string", "description": "The name of the S3 bucket. Read more here.", "order": 4 }, "bucket_prefix": { "title": "Target S3 Bucket Prefix", "type": "string", "description": "S3 prefix", "order": 5 }, "lakeformation_database_name": { "title": "Lake Formation Database Name", "type": "string", "description": "The default database this destination will use to create tables in per stream. Can be changed per connection by customizing the namespace.", "order": 6 }, "lakeformation_database_default_tag_key": { "title": "Lake Formation Database Tag Key", "description": "Add a default tag key to databases created by this destination", "examples": [ "pii_level" ], "type": "string", "order": 7, "x-speakeasy-param-sensitive": true }, "lakeformation_database_default_tag_values": { "title": "Lake Formation Database Tag Values", "description": "Add default values for the `Tag Key` to databases created by this destination. Comma separate for multiple values.", "examples": [ "private,public" ], "type": "string", "order": 8 }, "lakeformation_governed_tables": { "title": "Lake Formation Governed Tables", "description": "Whether to create tables as LF governed tables.", "type": "boolean", "default": false, "order": 9 }, "format": { "title": "Output Format *", "type": "object", "description": "Format of the data output.", "oneOf": [ { "title": "JSON Lines: Newline-delimited JSON", "required": [ "format_type" ], "properties": { "format_type": { "title": "Format Type *", "type": "string", "enum": [ "JSONL" ], "default": "JSONL" }, "compression_codec": { "title": "Compression Codec (Optional)", "description": "The compression algorithm used to compress data.", "type": "string", "enum": [ "UNCOMPRESSED", "GZIP" ], "default": "UNCOMPRESSED" } } }, { "title": "Parquet: Columnar Storage", "required": [ "format_type" ], "properties": { "format_type": { "title": "Format Type *", "type": "string", "enum": [ "Parquet" ], "default": "Parquet" }, "compression_codec": { "title": "Compression Codec (Optional)", "description": "The compression algorithm used to compress data.", "type": "string", "enum": [ "UNCOMPRESSED", "SNAPPY", "GZIP", "ZSTD" ], "default": "SNAPPY" } } } ], "order": 10 }, "partitioning": { "title": "Choose how to partition data", "description": "Partition data by cursor fields when a cursor field is a date", "type": "string", "enum": [ "NO PARTITIONING", "DATE", "YEAR", "MONTH", "DAY", "YEAR/MONTH", "YEAR/MONTH/DAY" ], "default": "NO PARTITIONING", "order": 11 }, "glue_catalog_float_as_decimal": { "title": "Glue Catalog: Float as Decimal", "description": "Cast float/double as decimal(38,18). This can help achieve higher accuracy and represent numbers correctly as received from the source.", "type": "boolean", "default": false, "order": 12 }, "destinationType": { "title": "aws-datalake", "const": "aws-datalake", "enum": [ "aws-datalake" ], "order": 0, "type": "string" } } }, "destination-milvus": { "title": "Destination Config", "description": "The configuration model for the Vector DB based destinations. This model is used to generate the UI for the destination configuration,\nas well as to provide type safety for the configuration passed to the destination.\n\nThe configuration model is composed of four parts:\n* Processing configuration\n* Embedding configuration\n* Indexing configuration\n* Advanced configuration\n\nProcessing, embedding and advanced configuration are provided by this base class, while the indexing configuration is provided by the destination connector in the sub class.", "type": "object", "properties": { "embedding": { "title": "Embedding", "description": "Embedding configuration", "group": "embedding", "type": "object", "oneOf": [ { "title": "OpenAI", "type": "object", "properties": { "mode": { "title": "Mode", "default": "openai", "const": "openai", "enum": [ "openai" ], "type": "string" }, "openai_key": { "title": "OpenAI API key", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true } }, "required": [ "openai_key", "mode" ], "description": "Use the OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." }, { "title": "Cohere", "type": "object", "properties": { "mode": { "title": "Mode", "default": "cohere", "const": "cohere", "enum": [ "cohere" ], "type": "string" }, "cohere_key": { "title": "Cohere API key", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true } }, "required": [ "cohere_key", "mode" ], "description": "Use the Cohere API to embed text." }, { "title": "Fake", "type": "object", "properties": { "mode": { "title": "Mode", "default": "fake", "const": "fake", "enum": [ "fake" ], "type": "string" } }, "description": "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs.", "required": [ "mode" ] }, { "title": "Azure OpenAI", "type": "object", "properties": { "mode": { "title": "Mode", "default": "azure_openai", "const": "azure_openai", "enum": [ "azure_openai" ], "type": "string" }, "openai_key": { "title": "Azure OpenAI API key", "description": "The API key for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true }, "api_base": { "title": "Resource base URL", "description": "The base URL for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", "examples": [ "https://your-resource-name.openai.azure.com" ], "type": "string" }, "deployment": { "title": "Deployment", "description": "The deployment for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", "examples": [ "your-resource-name" ], "type": "string" } }, "required": [ "openai_key", "api_base", "deployment", "mode" ], "description": "Use the Azure-hosted OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." }, { "title": "OpenAI-compatible", "type": "object", "properties": { "mode": { "title": "Mode", "default": "openai_compatible", "const": "openai_compatible", "enum": [ "openai_compatible" ], "type": "string" }, "api_key": { "title": "API key", "default": "", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true }, "base_url": { "title": "Base URL", "description": "The base URL for your OpenAI-compatible service", "examples": [ "https://your-service-name.com" ], "type": "string" }, "model_name": { "title": "Model name", "description": "The name of the model to use for embedding", "default": "text-embedding-ada-002", "examples": [ "text-embedding-ada-002" ], "type": "string" }, "dimensions": { "title": "Embedding dimensions", "description": "The number of dimensions the embedding model is generating", "examples": [ 1536, 384 ], "type": "integer" } }, "required": [ "base_url", "dimensions", "mode" ], "description": "Use a service that's compatible with the OpenAI API to embed text." } ] }, "processing": { "title": "ProcessingConfigModel", "type": "object", "properties": { "chunk_size": { "title": "Chunk size", "description": "Size of chunks in tokens to store in vector store (make sure it is not too big for the context if your LLM)", "maximum": 8191, "minimum": 1, "type": "integer" }, "chunk_overlap": { "title": "Chunk overlap", "description": "Size of overlap between chunks in tokens to store in vector store to better capture relevant context", "default": 0, "type": "integer" }, "text_fields": { "title": "Text fields to embed", "description": "List of fields in the record that should be used to calculate the embedding. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered text fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.", "default": [], "always_show": true, "examples": [ "text", "user.name", "users.*.name" ], "type": "array", "items": { "type": "string" } }, "metadata_fields": { "title": "Fields to store as metadata", "description": "List of fields in the record that should be stored as metadata. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered metadata fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. When specifying nested paths, all matching values are flattened into an array set to a field named by the path.", "default": [], "always_show": true, "examples": [ "age", "user", "user.name" ], "type": "array", "items": { "type": "string" } }, "text_splitter": { "title": "Text splitter", "description": "Split text fields into chunks based on the specified method.", "type": "object", "oneOf": [ { "title": "By Separator", "type": "object", "properties": { "mode": { "title": "Mode", "default": "separator", "const": "separator", "enum": [ "separator" ], "type": "string" }, "separators": { "title": "Separators", "description": "List of separator strings to split text fields by. The separator itself needs to be wrapped in double quotes, e.g. to split by the dot character, use \".\". To split by a newline, use \"\\n\".", "default": [ "\"\\n\\n\"", "\"\\n\"", "\" \"", "\"\"" ], "type": "array", "items": { "type": "string" } }, "keep_separator": { "title": "Keep separator", "description": "Whether to keep the separator in the resulting chunks", "default": false, "type": "boolean" } }, "description": "Split the text by the list of separators until the chunk size is reached, using the earlier mentioned separators where possible. This is useful for splitting text fields by paragraphs, sentences, words, etc.", "required": [ "mode" ] }, { "title": "By Markdown header", "type": "object", "properties": { "mode": { "title": "Mode", "default": "markdown", "const": "markdown", "enum": [ "markdown" ], "type": "string" }, "split_level": { "title": "Split level", "description": "Level of markdown headers to split text fields by. Headings down to the specified level will be used as split points", "default": 1, "minimum": 1, "maximum": 6, "type": "integer" } }, "description": "Split the text by Markdown headers down to the specified header level. If the chunk size fits multiple sections, they will be combined into a single chunk.", "required": [ "mode" ] }, { "title": "By Programming Language", "type": "object", "properties": { "mode": { "title": "Mode", "default": "code", "const": "code", "enum": [ "code" ], "type": "string" }, "language": { "title": "Language", "description": "Split code in suitable places based on the programming language", "enum": [ "cpp", "go", "java", "js", "php", "proto", "python", "rst", "ruby", "rust", "scala", "swift", "markdown", "latex", "html", "sol" ], "type": "string" } }, "required": [ "language", "mode" ], "description": "Split the text by suitable delimiters based on the programming language. This is useful for splitting code into chunks." } ] }, "field_name_mappings": { "title": "Field name mappings", "description": "List of fields to rename. Not applicable for nested fields, but can be used to rename fields already flattened via dot notation.", "default": [], "type": "array", "items": { "title": "FieldNameMappingConfigModel", "type": "object", "properties": { "from_field": { "title": "From field name", "description": "The field name in the source", "type": "string" }, "to_field": { "title": "To field name", "description": "The field name to use in the destination", "type": "string" } }, "required": [ "from_field", "to_field" ] } } }, "required": [ "chunk_size" ], "group": "processing" }, "omit_raw_text": { "title": "Do not store raw text", "description": "Do not store the text that gets embedded along with the vector and the metadata in the destination. If set to true, only the vector and the metadata will be stored - in this case raw text for LLM use cases needs to be retrieved from another source.", "default": false, "group": "advanced", "type": "boolean" }, "indexing": { "title": "Indexing", "type": "object", "properties": { "host": { "title": "Public Endpoint", "description": "The public endpoint of the Milvus instance. ", "order": 1, "examples": [ "https://my-instance.zone.zillizcloud.com", "tcp://host.docker.internal:19530", "tcp://my-local-milvus:19530" ], "type": "string" }, "db": { "title": "Database Name", "description": "The database to connect to", "default": "", "type": "string" }, "collection": { "title": "Collection Name", "description": "The collection to load data into", "order": 3, "type": "string" }, "auth": { "title": "Authentication", "description": "Authentication method", "type": "object", "order": 2, "oneOf": [ { "title": "API Token", "type": "object", "properties": { "mode": { "title": "Mode", "default": "token", "const": "token", "enum": [ "token" ], "type": "string" }, "token": { "title": "API Token", "description": "API Token for the Milvus instance", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true } }, "required": [ "token", "mode" ], "description": "Authenticate using an API token (suitable for Zilliz Cloud)" }, { "title": "Username/Password", "type": "object", "properties": { "mode": { "title": "Mode", "default": "username_password", "const": "username_password", "enum": [ "username_password" ], "type": "string" }, "username": { "title": "Username", "description": "Username for the Milvus instance", "order": 1, "type": "string" }, "password": { "title": "Password", "description": "Password for the Milvus instance", "airbyte_secret": true, "order": 2, "type": "string", "x-speakeasy-param-sensitive": true } }, "required": [ "username", "password", "mode" ], "description": "Authenticate using username and password (suitable for self-managed Milvus clusters)" }, { "title": "No auth", "type": "object", "properties": { "mode": { "title": "Mode", "default": "no_auth", "const": "no_auth", "enum": [ "no_auth" ], "type": "string" } }, "description": "Do not authenticate (suitable for locally running test clusters, do not use for clusters with public IP addresses)", "required": [ "mode" ] } ] }, "vector_field": { "title": "Vector Field", "description": "The field in the entity that contains the vector", "default": "vector", "type": "string" }, "text_field": { "title": "Text Field", "description": "The field in the entity that contains the embedded text", "default": "text", "type": "string" } }, "required": [ "host", "collection", "auth" ], "group": "indexing", "description": "Indexing configuration" }, "destinationType": { "title": "milvus", "const": "milvus", "enum": [ "milvus" ], "order": 0, "type": "string" } }, "required": [ "embedding", "processing", "indexing", "destinationType" ], "groups": [ { "id": "processing", "title": "Processing" }, { "id": "embedding", "title": "Embedding" }, { "id": "indexing", "title": "Indexing" }, { "id": "advanced", "title": "Advanced" } ] }, "destination-firebolt": { "title": "Firebolt Spec", "type": "object", "required": [ "username", "password", "database", "destinationType" ], "properties": { "username": { "type": "string", "title": "Username", "description": "Firebolt email address you use to login.", "examples": [ "username@email.com" ], "order": 0 }, "password": { "type": "string", "title": "Password", "description": "Firebolt password.", "airbyte_secret": true, "order": 1, "x-speakeasy-param-sensitive": true }, "account": { "type": "string", "title": "Account", "description": "Firebolt account to login." }, "host": { "type": "string", "title": "Host", "description": "The host name of your Firebolt database.", "examples": [ "api.app.firebolt.io" ] }, "database": { "type": "string", "title": "Database", "description": "The database to connect to." }, "engine": { "type": "string", "title": "Engine", "description": "Engine name or url to connect to." }, "loading_method": { "type": "object", "title": "Loading Method", "description": "Loading method used to select the way data will be uploaded to Firebolt", "oneOf": [ { "title": "SQL Inserts", "additionalProperties": false, "required": [ "method" ], "properties": { "method": { "type": "string", "const": "SQL", "enum": [ "SQL" ] } } }, { "title": "External Table via S3", "additionalProperties": false, "required": [ "method", "s3_bucket", "s3_region", "aws_key_id", "aws_key_secret" ], "properties": { "method": { "type": "string", "const": "S3", "enum": [ "S3" ] }, "s3_bucket": { "type": "string", "title": "S3 bucket name", "description": "The name of the S3 bucket." }, "s3_region": { "type": "string", "title": "S3 region name", "description": "Region name of the S3 bucket.", "examples": [ "us-east-1" ] }, "aws_key_id": { "type": "string", "title": "AWS Key ID", "airbyte_secret": true, "description": "AWS access key granting read and write access to S3.", "x-speakeasy-param-sensitive": true }, "aws_key_secret": { "type": "string", "title": "AWS Key Secret", "airbyte_secret": true, "description": "Corresponding secret part of the AWS Key", "x-speakeasy-param-sensitive": true } } } ] }, "destinationType": { "title": "firebolt", "const": "firebolt", "enum": [ "firebolt" ], "order": 0, "type": "string" } } }, "destination-google-sheets": { "title": "Destination Google Sheets", "type": "object", "required": [ "spreadsheet_id", "credentials", "destinationType" ], "properties": { "spreadsheet_id": { "type": "string", "title": "Spreadsheet Link", "description": "The link to your spreadsheet. See this guide for more details.", "examples": [ "https://docs.google.com/spreadsheets/d/1hLd9Qqti3UyLXZB2aFfUWDT7BG/edit" ] }, "credentials": { "type": "object", "title": "Authentication via Google (OAuth)", "description": "Google API Credentials for connecting to Google Sheets and Google Drive APIs", "required": [ "client_id", "client_secret", "refresh_token" ], "properties": { "client_id": { "title": "Client ID", "type": "string", "description": "The Client ID of your Google Sheets developer application.", "airbyte_secret": true }, "client_secret": { "title": "Client Secret", "type": "string", "description": "The Client Secret of your Google Sheets developer application.", "airbyte_secret": true }, "refresh_token": { "title": "Refresh Token", "type": "string", "description": "The token for obtaining new access token.", "airbyte_secret": true, "x-speakeasy-param-sensitive": true } } }, "destinationType": { "title": "google-sheets", "const": "google-sheets", "enum": [ "google-sheets" ], "order": 0, "type": "string" } } }, "destination-astra": { "title": "Destination Config", "description": "The configuration model for the Vector DB based destinations. This model is used to generate the UI for the destination configuration,\nas well as to provide type safety for the configuration passed to the destination.\n\nThe configuration model is composed of four parts:\n* Processing configuration\n* Embedding configuration\n* Indexing configuration\n* Advanced configuration\n\nProcessing, embedding and advanced configuration are provided by this base class, while the indexing configuration is provided by the destination connector in the sub class.", "type": "object", "properties": { "embedding": { "title": "Embedding", "description": "Embedding configuration", "group": "embedding", "type": "object", "oneOf": [ { "title": "OpenAI", "type": "object", "properties": { "mode": { "title": "Mode", "default": "openai", "const": "openai", "enum": [ "openai" ], "type": "string" }, "openai_key": { "title": "OpenAI API key", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true } }, "required": [ "openai_key", "mode" ], "description": "Use the OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." }, { "title": "Cohere", "type": "object", "properties": { "mode": { "title": "Mode", "default": "cohere", "const": "cohere", "enum": [ "cohere" ], "type": "string" }, "cohere_key": { "title": "Cohere API key", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true } }, "required": [ "cohere_key", "mode" ], "description": "Use the Cohere API to embed text." }, { "title": "Fake", "type": "object", "properties": { "mode": { "title": "Mode", "default": "fake", "const": "fake", "enum": [ "fake" ], "type": "string" } }, "description": "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs.", "required": [ "mode" ] }, { "title": "Azure OpenAI", "type": "object", "properties": { "mode": { "title": "Mode", "default": "azure_openai", "const": "azure_openai", "enum": [ "azure_openai" ], "type": "string" }, "openai_key": { "title": "Azure OpenAI API key", "description": "The API key for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true }, "api_base": { "title": "Resource base URL", "description": "The base URL for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", "examples": [ "https://your-resource-name.openai.azure.com" ], "type": "string" }, "deployment": { "title": "Deployment", "description": "The deployment for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", "examples": [ "your-resource-name" ], "type": "string" } }, "required": [ "openai_key", "api_base", "deployment", "mode" ], "description": "Use the Azure-hosted OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." }, { "title": "OpenAI-compatible", "type": "object", "properties": { "mode": { "title": "Mode", "default": "openai_compatible", "const": "openai_compatible", "enum": [ "openai_compatible" ], "type": "string" }, "api_key": { "title": "API key", "default": "", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true }, "base_url": { "title": "Base URL", "description": "The base URL for your OpenAI-compatible service", "examples": [ "https://your-service-name.com" ], "type": "string" }, "model_name": { "title": "Model name", "description": "The name of the model to use for embedding", "default": "text-embedding-ada-002", "examples": [ "text-embedding-ada-002" ], "type": "string" }, "dimensions": { "title": "Embedding dimensions", "description": "The number of dimensions the embedding model is generating", "examples": [ 1536, 384 ], "type": "integer" } }, "required": [ "base_url", "dimensions", "mode" ], "description": "Use a service that's compatible with the OpenAI API to embed text." } ] }, "processing": { "title": "ProcessingConfigModel", "type": "object", "properties": { "chunk_size": { "title": "Chunk size", "description": "Size of chunks in tokens to store in vector store (make sure it is not too big for the context if your LLM)", "maximum": 8191, "minimum": 1, "type": "integer" }, "chunk_overlap": { "title": "Chunk overlap", "description": "Size of overlap between chunks in tokens to store in vector store to better capture relevant context", "default": 0, "type": "integer" }, "text_fields": { "title": "Text fields to embed", "description": "List of fields in the record that should be used to calculate the embedding. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered text fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.", "default": [], "always_show": true, "examples": [ "text", "user.name", "users.*.name" ], "type": "array", "items": { "type": "string" } }, "metadata_fields": { "title": "Fields to store as metadata", "description": "List of fields in the record that should be stored as metadata. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered metadata fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. When specifying nested paths, all matching values are flattened into an array set to a field named by the path.", "default": [], "always_show": true, "examples": [ "age", "user", "user.name" ], "type": "array", "items": { "type": "string" } }, "text_splitter": { "title": "Text splitter", "description": "Split text fields into chunks based on the specified method.", "type": "object", "oneOf": [ { "title": "By Separator", "type": "object", "properties": { "mode": { "title": "Mode", "default": "separator", "const": "separator", "enum": [ "separator" ], "type": "string" }, "separators": { "title": "Separators", "description": "List of separator strings to split text fields by. The separator itself needs to be wrapped in double quotes, e.g. to split by the dot character, use \".\". To split by a newline, use \"\\n\".", "default": [ "\"\\n\\n\"", "\"\\n\"", "\" \"", "\"\"" ], "type": "array", "items": { "type": "string" } }, "keep_separator": { "title": "Keep separator", "description": "Whether to keep the separator in the resulting chunks", "default": false, "type": "boolean" } }, "description": "Split the text by the list of separators until the chunk size is reached, using the earlier mentioned separators where possible. This is useful for splitting text fields by paragraphs, sentences, words, etc.", "required": [ "mode" ] }, { "title": "By Markdown header", "type": "object", "properties": { "mode": { "title": "Mode", "default": "markdown", "const": "markdown", "enum": [ "markdown" ], "type": "string" }, "split_level": { "title": "Split level", "description": "Level of markdown headers to split text fields by. Headings down to the specified level will be used as split points", "default": 1, "minimum": 1, "maximum": 6, "type": "integer" } }, "description": "Split the text by Markdown headers down to the specified header level. If the chunk size fits multiple sections, they will be combined into a single chunk.", "required": [ "mode" ] }, { "title": "By Programming Language", "type": "object", "properties": { "mode": { "title": "Mode", "default": "code", "const": "code", "enum": [ "code" ], "type": "string" }, "language": { "title": "Language", "description": "Split code in suitable places based on the programming language", "enum": [ "cpp", "go", "java", "js", "php", "proto", "python", "rst", "ruby", "rust", "scala", "swift", "markdown", "latex", "html", "sol" ], "type": "string" } }, "required": [ "language", "mode" ], "description": "Split the text by suitable delimiters based on the programming language. This is useful for splitting code into chunks." } ] }, "field_name_mappings": { "title": "Field name mappings", "description": "List of fields to rename. Not applicable for nested fields, but can be used to rename fields already flattened via dot notation.", "default": [], "type": "array", "items": { "title": "FieldNameMappingConfigModel", "type": "object", "properties": { "from_field": { "title": "From field name", "description": "The field name in the source", "type": "string" }, "to_field": { "title": "To field name", "description": "The field name to use in the destination", "type": "string" } }, "required": [ "from_field", "to_field" ] } } }, "required": [ "chunk_size" ], "group": "processing" }, "omit_raw_text": { "title": "Do not store raw text", "description": "Do not store the text that gets embedded along with the vector and the metadata in the destination. If set to true, only the vector and the metadata will be stored - in this case raw text for LLM use cases needs to be retrieved from another source.", "default": false, "group": "advanced", "type": "boolean" }, "indexing": { "title": "Indexing", "type": "object", "properties": { "astra_db_app_token": { "title": "Astra DB Application Token", "description": "The application token authorizes a user to connect to a specific Astra DB database. It is created when the user clicks the Generate Token button on the Overview tab of the Database page in the Astra UI.", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true }, "astra_db_endpoint": { "title": "Astra DB Endpoint", "description": "The endpoint specifies which Astra DB database queries are sent to. It can be copied from the Database Details section of the Overview tab of the Database page in the Astra UI.", "pattern": "^https:\\/\\/([a-z]|[0-9]){8}-([a-z]|[0-9]){4}-([a-z]|[0-9]){4}-([a-z]|[0-9]){4}-([a-z]|[0-9]){12}-[^\\.]*?\\.apps\\.astra\\.datastax\\.com", "examples": [ "https://8292d414-dd1b-4c33-8431-e838bedc04f7-us-east1.apps.astra.datastax.com" ], "type": "string" }, "astra_db_keyspace": { "title": "Astra DB Keyspace", "description": "Keyspaces (or Namespaces) serve as containers for organizing data within a database. You can create a new keyspace uisng the Data Explorer tab in the Astra UI. The keyspace default_keyspace is created for you when you create a Vector Database in Astra DB.", "type": "string", "x-speakeasy-param-sensitive": true }, "collection": { "title": "Astra DB collection", "description": "Collections hold data. They are analagous to tables in traditional Cassandra terminology. This tool will create the collection with the provided name automatically if it does not already exist. Alternatively, you can create one thorugh the Data Explorer tab in the Astra UI.", "type": "string" } }, "required": [ "astra_db_app_token", "astra_db_endpoint", "astra_db_keyspace", "collection" ], "description": "Astra DB gives developers the APIs, real-time data and ecosystem integrations to put accurate RAG and Gen AI apps with fewer hallucinations in production.", "group": "indexing" }, "destinationType": { "title": "astra", "const": "astra", "enum": [ "astra" ], "order": 0, "type": "string" } }, "required": [ "embedding", "processing", "indexing", "destinationType" ], "groups": [ { "id": "processing", "title": "Processing" }, { "id": "embedding", "title": "Embedding" }, { "id": "indexing", "title": "Indexing" }, { "id": "advanced", "title": "Advanced" } ] }, "destination-databend": { "title": "Destination Databend", "type": "object", "required": [ "host", "username", "database", "destinationType" ], "properties": { "host": { "title": "Host", "description": "Hostname of the database.", "type": "string", "order": 0 }, "port": { "title": "Port", "description": "Port of the database.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 443, "examples": [ "443" ], "order": 2 }, "database": { "title": "DB Name", "description": "Name of the database.", "type": "string", "order": 3 }, "table": { "title": "Default Table", "description": "The default table was written to.", "type": "string", "examples": [ "default" ], "default": "default", "order": 4 }, "username": { "title": "User", "description": "Username to use to access the database.", "type": "string", "order": 5 }, "password": { "title": "Password", "description": "Password associated with the username.", "type": "string", "airbyte_secret": true, "order": 6, "x-speakeasy-param-sensitive": true }, "destinationType": { "title": "databend", "const": "databend", "enum": [ "databend" ], "order": 0, "type": "string" } } }, "destination-teradata": { "title": "Teradata Destination Spec", "type": "object", "required": [ "host", "username", "destinationType" ], "properties": { "host": { "title": "Host", "description": "Hostname of the database.", "type": "string", "order": 0 }, "username": { "title": "User", "description": "Username to use to access the database.", "type": "string", "order": 1 }, "password": { "title": "Password", "description": "Password associated with the username.", "type": "string", "airbyte_secret": true, "order": 2, "x-speakeasy-param-sensitive": true }, "schema": { "title": "Default Schema", "description": "The default schema tables are written to if the source does not specify a namespace. The usual value for this field is \"public\".", "type": "string", "examples": [ "airbyte_td" ], "default": "airbyte_td", "order": 3 }, "ssl": { "title": "SSL Connection", "description": "Encrypt data using SSL. When activating SSL, please select one of the connection modes.", "type": "boolean", "default": false, "order": 5 }, "ssl_mode": { "title": "SSL modes", "description": "SSL connection modes. \n disable - Chose this mode to disable encryption of communication between Airbyte and destination database\n allow - Chose this mode to enable encryption only when required by the destination database\n prefer - Chose this mode to allow unencrypted connection only if the destination database does not support encryption\n require - Chose this mode to always require encryption. If the destination database server does not support encryption, connection will fail\n verify-ca - Chose this mode to always require encryption and to verify that the destination database server has a valid SSL certificate\n verify-full - This is the most secure mode. Chose this mode to always require encryption and to verify the identity of the destination database server\n See more information - in the docs.", "type": "object", "order": 6, "oneOf": [ { "title": "disable", "additionalProperties": false, "description": "Disable SSL.", "required": [ "mode" ], "properties": { "mode": { "type": "string", "const": "disable", "enum": [ "disable" ], "default": "disable", "order": 0 } } }, { "title": "allow", "additionalProperties": false, "description": "Allow SSL mode.", "required": [ "mode" ], "properties": { "mode": { "type": "string", "const": "allow", "enum": [ "allow" ], "default": "allow", "order": 0 } } }, { "title": "prefer", "additionalProperties": false, "description": "Prefer SSL mode.", "required": [ "mode" ], "properties": { "mode": { "type": "string", "const": "prefer", "enum": [ "prefer" ], "default": "prefer", "order": 0 } } }, { "title": "require", "additionalProperties": false, "description": "Require SSL mode.", "required": [ "mode" ], "properties": { "mode": { "type": "string", "const": "require", "enum": [ "require" ], "default": "require", "order": 0 } } }, { "title": "verify-ca", "additionalProperties": false, "description": "Verify-ca SSL mode.", "required": [ "mode", "ssl_ca_certificate" ], "properties": { "mode": { "type": "string", "const": "verify-ca", "enum": [ "verify-ca" ], "default": "verify-ca", "order": 0 }, "ssl_ca_certificate": { "type": "string", "title": "CA certificate", "description": "Specifies the file name of a PEM file that contains Certificate Authority (CA) certificates for use with SSLMODE=verify-ca.\n See more information - in the docs.", "airbyte_secret": true, "multiline": true, "order": 1 } } }, { "title": "verify-full", "additionalProperties": false, "description": "Verify-full SSL mode.", "required": [ "mode", "ssl_ca_certificate" ], "properties": { "mode": { "type": "string", "const": "verify-full", "enum": [ "verify-full" ], "default": "verify-full", "order": 0 }, "ssl_ca_certificate": { "type": "string", "title": "CA certificate", "description": "Specifies the file name of a PEM file that contains Certificate Authority (CA) certificates for use with SSLMODE=verify-full.\n See more information - in the docs.", "airbyte_secret": true, "multiline": true, "order": 1 } } } ] }, "jdbc_url_params": { "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "title": "JDBC URL Params", "type": "string", "order": 7 }, "destinationType": { "title": "teradata", "const": "teradata", "enum": [ "teradata" ], "order": 0, "type": "string" } } }, "destination-pinecone": { "title": "Destination Config", "description": "The configuration model for the Vector DB based destinations. This model is used to generate the UI for the destination configuration,\nas well as to provide type safety for the configuration passed to the destination.\n\nThe configuration model is composed of four parts:\n* Processing configuration\n* Embedding configuration\n* Indexing configuration\n* Advanced configuration\n\nProcessing, embedding and advanced configuration are provided by this base class, while the indexing configuration is provided by the destination connector in the sub class.", "type": "object", "properties": { "embedding": { "title": "Embedding", "description": "Embedding configuration", "group": "embedding", "type": "object", "oneOf": [ { "title": "OpenAI", "type": "object", "properties": { "mode": { "title": "Mode", "default": "openai", "const": "openai", "enum": [ "openai" ], "type": "string" }, "openai_key": { "title": "OpenAI API key", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true } }, "required": [ "openai_key", "mode" ], "description": "Use the OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." }, { "title": "Cohere", "type": "object", "properties": { "mode": { "title": "Mode", "default": "cohere", "const": "cohere", "enum": [ "cohere" ], "type": "string" }, "cohere_key": { "title": "Cohere API key", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true } }, "required": [ "cohere_key", "mode" ], "description": "Use the Cohere API to embed text." }, { "title": "Fake", "type": "object", "properties": { "mode": { "title": "Mode", "default": "fake", "const": "fake", "enum": [ "fake" ], "type": "string" } }, "description": "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs.", "required": [ "mode" ] }, { "title": "Azure OpenAI", "type": "object", "properties": { "mode": { "title": "Mode", "default": "azure_openai", "const": "azure_openai", "enum": [ "azure_openai" ], "type": "string" }, "openai_key": { "title": "Azure OpenAI API key", "description": "The API key for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true }, "api_base": { "title": "Resource base URL", "description": "The base URL for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", "examples": [ "https://your-resource-name.openai.azure.com" ], "type": "string" }, "deployment": { "title": "Deployment", "description": "The deployment for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", "examples": [ "your-resource-name" ], "type": "string" } }, "required": [ "openai_key", "api_base", "deployment", "mode" ], "description": "Use the Azure-hosted OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." }, { "title": "OpenAI-compatible", "type": "object", "properties": { "mode": { "title": "Mode", "default": "openai_compatible", "const": "openai_compatible", "enum": [ "openai_compatible" ], "type": "string" }, "api_key": { "title": "API key", "default": "", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true }, "base_url": { "title": "Base URL", "description": "The base URL for your OpenAI-compatible service", "examples": [ "https://your-service-name.com" ], "type": "string" }, "model_name": { "title": "Model name", "description": "The name of the model to use for embedding", "default": "text-embedding-ada-002", "examples": [ "text-embedding-ada-002" ], "type": "string" }, "dimensions": { "title": "Embedding dimensions", "description": "The number of dimensions the embedding model is generating", "examples": [ 1536, 384 ], "type": "integer" } }, "required": [ "base_url", "dimensions", "mode" ], "description": "Use a service that's compatible with the OpenAI API to embed text." } ] }, "processing": { "title": "ProcessingConfigModel", "type": "object", "properties": { "chunk_size": { "title": "Chunk size", "description": "Size of chunks in tokens to store in vector store (make sure it is not too big for the context if your LLM)", "maximum": 8191, "minimum": 1, "type": "integer" }, "chunk_overlap": { "title": "Chunk overlap", "description": "Size of overlap between chunks in tokens to store in vector store to better capture relevant context", "default": 0, "type": "integer" }, "text_fields": { "title": "Text fields to embed", "description": "List of fields in the record that should be used to calculate the embedding. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered text fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.", "default": [], "always_show": true, "examples": [ "text", "user.name", "users.*.name" ], "type": "array", "items": { "type": "string" } }, "metadata_fields": { "title": "Fields to store as metadata", "description": "List of fields in the record that should be stored as metadata. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered metadata fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. When specifying nested paths, all matching values are flattened into an array set to a field named by the path.", "default": [], "always_show": true, "examples": [ "age", "user", "user.name" ], "type": "array", "items": { "type": "string" } }, "text_splitter": { "title": "Text splitter", "description": "Split text fields into chunks based on the specified method.", "type": "object", "oneOf": [ { "title": "By Separator", "type": "object", "properties": { "mode": { "title": "Mode", "default": "separator", "const": "separator", "enum": [ "separator" ], "type": "string" }, "separators": { "title": "Separators", "description": "List of separator strings to split text fields by. The separator itself needs to be wrapped in double quotes, e.g. to split by the dot character, use \".\". To split by a newline, use \"\\n\".", "default": [ "\"\\n\\n\"", "\"\\n\"", "\" \"", "\"\"" ], "type": "array", "items": { "type": "string" } }, "keep_separator": { "title": "Keep separator", "description": "Whether to keep the separator in the resulting chunks", "default": false, "type": "boolean" } }, "description": "Split the text by the list of separators until the chunk size is reached, using the earlier mentioned separators where possible. This is useful for splitting text fields by paragraphs, sentences, words, etc.", "required": [ "mode" ] }, { "title": "By Markdown header", "type": "object", "properties": { "mode": { "title": "Mode", "default": "markdown", "const": "markdown", "enum": [ "markdown" ], "type": "string" }, "split_level": { "title": "Split level", "description": "Level of markdown headers to split text fields by. Headings down to the specified level will be used as split points", "default": 1, "minimum": 1, "maximum": 6, "type": "integer" } }, "description": "Split the text by Markdown headers down to the specified header level. If the chunk size fits multiple sections, they will be combined into a single chunk.", "required": [ "mode" ] }, { "title": "By Programming Language", "type": "object", "properties": { "mode": { "title": "Mode", "default": "code", "const": "code", "enum": [ "code" ], "type": "string" }, "language": { "title": "Language", "description": "Split code in suitable places based on the programming language", "enum": [ "cpp", "go", "java", "js", "php", "proto", "python", "rst", "ruby", "rust", "scala", "swift", "markdown", "latex", "html", "sol" ], "type": "string" } }, "required": [ "language", "mode" ], "description": "Split the text by suitable delimiters based on the programming language. This is useful for splitting code into chunks." } ] }, "field_name_mappings": { "title": "Field name mappings", "description": "List of fields to rename. Not applicable for nested fields, but can be used to rename fields already flattened via dot notation.", "default": [], "type": "array", "items": { "title": "FieldNameMappingConfigModel", "type": "object", "properties": { "from_field": { "title": "From field name", "description": "The field name in the source", "type": "string" }, "to_field": { "title": "To field name", "description": "The field name to use in the destination", "type": "string" } }, "required": [ "from_field", "to_field" ] } } }, "required": [ "chunk_size" ], "group": "processing" }, "omit_raw_text": { "title": "Do not store raw text", "description": "Do not store the text that gets embedded along with the vector and the metadata in the destination. If set to true, only the vector and the metadata will be stored - in this case raw text for LLM use cases needs to be retrieved from another source.", "default": false, "group": "advanced", "type": "boolean" }, "indexing": { "title": "Indexing", "type": "object", "properties": { "pinecone_key": { "title": "Pinecone API key", "description": "The Pinecone API key to use matching the environment (copy from Pinecone console)", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true }, "pinecone_environment": { "title": "Pinecone Environment", "description": "Pinecone Cloud environment to use", "examples": [ "us-west1-gcp", "gcp-starter" ], "type": "string" }, "index": { "title": "Index", "description": "Pinecone index in your project to load data into", "type": "string" } }, "required": [ "pinecone_key", "pinecone_environment", "index" ], "description": "Pinecone is a popular vector store that can be used to store and retrieve embeddings.", "group": "indexing" }, "destinationType": { "title": "pinecone", "const": "pinecone", "enum": [ "pinecone" ], "order": 0, "type": "string" } }, "required": [ "embedding", "processing", "indexing", "destinationType" ], "groups": [ { "id": "processing", "title": "Processing" }, { "id": "embedding", "title": "Embedding" }, { "id": "indexing", "title": "Indexing" }, { "id": "advanced", "title": "Advanced" } ] }, "destination-duckdb": { "title": "Destination Duckdb", "type": "object", "required": [ "destination_path", "destinationType" ], "properties": { "motherduck_api_key": { "title": "MotherDuck API Key", "type": "string", "description": "API key to use for authentication to a MotherDuck database.", "airbyte_secret": true, "x-speakeasy-param-sensitive": true }, "destination_path": { "title": "Destination DB", "type": "string", "description": "Path to the .duckdb file, or the text 'md:' to connect to MotherDuck. The file will be placed inside that local mount. For more information check out our docs", "examples": [ "/local/destination.duckdb", "md:", "motherduck:" ] }, "schema": { "title": "Destination Schema", "type": "string", "description": "Database schema name, default for duckdb is 'main'.", "example": "main" }, "destinationType": { "title": "duckdb", "const": "duckdb", "enum": [ "duckdb" ], "order": 0, "type": "string" } } }, "destination-sftp-json": { "title": "Destination SFTP JSON", "type": "object", "required": [ "host", "username", "password", "destination_path", "destinationType" ], "properties": { "host": { "title": "Host", "description": "Hostname of the SFTP server.", "type": "string", "order": 0 }, "port": { "title": "Port", "description": "Port of the SFTP server.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 22, "examples": [ 22 ], "order": 1 }, "username": { "title": "User", "description": "Username to use to access the SFTP server.", "type": "string", "order": 2 }, "password": { "title": "Password", "description": "Password associated with the username.", "type": "string", "airbyte_secret": true, "order": 3, "x-speakeasy-param-sensitive": true }, "destination_path": { "title": "Destination path", "type": "string", "description": "Path to the directory where json files will be written.", "examples": [ "/json_data" ], "order": 4 }, "destinationType": { "title": "sftp-json", "const": "sftp-json", "enum": [ "sftp-json" ], "order": 0, "type": "string" } } }, "destination-kinesis": { "title": "Kinesis Destination Spec", "type": "object", "required": [ "endpoint", "region", "shardCount", "accessKey", "privateKey", "bufferSize", "destinationType" ], "properties": { "endpoint": { "title": "Endpoint", "description": "AWS Kinesis endpoint.", "type": "string", "examples": [ "kinesis.us‑west‑1.amazonaws.com" ], "order": 0 }, "region": { "title": "Region", "description": "AWS region. Your account determines the Regions that are available to you.", "type": "string", "examples": [ "us‑west‑1" ], "order": 1 }, "shardCount": { "title": "Shard Count", "description": "Number of shards to which the data should be streamed.", "type": "integer", "default": 5, "order": 2 }, "accessKey": { "title": "Access Key", "description": "Generate the AWS Access Key for current user.", "airbyte_secret": true, "type": "string", "order": 3 }, "privateKey": { "title": "Private Key", "description": "The AWS Private Key - a string of numbers and letters that are unique for each account, also known as a \"recovery phrase\".", "airbyte_secret": true, "type": "string", "order": 4 }, "bufferSize": { "title": "Buffer Size", "description": "Buffer size for storing kinesis records before being batch streamed.", "type": "integer", "minimum": 1, "maximum": 500, "default": 100, "order": 5 }, "destinationType": { "title": "kinesis", "const": "kinesis", "enum": [ "kinesis" ], "order": 0, "type": "string" } } }, "destination-s3": { "title": "S3 Destination Spec", "type": "object", "required": [ "s3_bucket_name", "s3_bucket_path", "s3_bucket_region", "format", "destinationType" ], "properties": { "access_key_id": { "type": "string", "description": "The access key ID to access the S3 bucket. Airbyte requires Read and Write permissions to the given bucket. Read more here.", "title": "S3 Key ID", "airbyte_secret": true, "always_show": true, "examples": [ "A012345678910EXAMPLE" ], "order": 0, "x-speakeasy-param-sensitive": true }, "secret_access_key": { "type": "string", "description": "The corresponding secret to the access key ID. Read more here", "title": "S3 Access Key", "airbyte_secret": true, "always_show": true, "examples": [ "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY" ], "order": 1, "x-speakeasy-param-sensitive": true }, "s3_bucket_name": { "title": "S3 Bucket Name", "type": "string", "description": "The name of the S3 bucket. Read more here.", "examples": [ "airbyte_sync" ], "order": 2 }, "s3_bucket_path": { "title": "S3 Bucket Path", "description": "Directory under the S3 bucket where data will be written. Read more here", "type": "string", "examples": [ "data_sync/test" ], "order": 3 }, "s3_bucket_region": { "title": "S3 Bucket Region", "type": "string", "default": "", "description": "The region of the S3 bucket. See here for all region codes.", "enum": [ "", "af-south-1", "ap-east-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-south-2", "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", "ap-southeast-4", "ca-central-1", "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", "eu-central-2", "eu-north-1", "eu-south-1", "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", "il-central-1", "me-central-1", "me-south-1", "sa-east-1", "us-east-1", "us-east-2", "us-gov-east-1", "us-gov-west-1", "us-west-1", "us-west-2" ], "order": 4 }, "format": { "title": "Output Format", "type": "object", "description": "Format of the data output. See here for more details", "oneOf": [ { "title": "CSV: Comma-Separated Values", "required": [ "format_type", "flattening" ], "properties": { "format_type": { "title": "Format Type", "type": "string", "enum": [ "CSV" ], "default": "CSV" }, "flattening": { "type": "string", "title": "Flattening", "description": "Whether the input json data should be normalized (flattened) in the output CSV. Please refer to docs for details.", "default": "No flattening", "enum": [ "No flattening", "Root level flattening" ] }, "compression": { "title": "Compression", "type": "object", "description": "Whether the output files should be compressed. If compression is selected, the output filename will have an extra extension (GZIP: \".csv.gz\").", "oneOf": [ { "title": "No Compression", "requires": [ "compression_type" ], "properties": { "compression_type": { "type": "string", "enum": [ "No Compression" ], "default": "No Compression" } } }, { "title": "GZIP", "requires": [ "compression_type" ], "properties": { "compression_type": { "type": "string", "enum": [ "GZIP" ], "default": "GZIP" } } } ] } } }, { "title": "JSON Lines: Newline-delimited JSON", "required": [ "format_type" ], "properties": { "format_type": { "title": "Format Type", "type": "string", "enum": [ "JSONL" ], "default": "JSONL" }, "flattening": { "type": "string", "title": "Flattening", "description": "Whether the input json data should be normalized (flattened) in the output JSON Lines. Please refer to docs for details.", "default": "No flattening", "enum": [ "No flattening", "Root level flattening" ] }, "compression": { "title": "Compression", "type": "object", "description": "Whether the output files should be compressed. If compression is selected, the output filename will have an extra extension (GZIP: \".jsonl.gz\").", "oneOf": [ { "title": "No Compression", "requires": "compression_type", "properties": { "compression_type": { "type": "string", "enum": [ "No Compression" ], "default": "No Compression" } } }, { "title": "GZIP", "requires": "compression_type", "properties": { "compression_type": { "type": "string", "enum": [ "GZIP" ], "default": "GZIP" } } } ] } } }, { "title": "Avro: Apache Avro", "required": [ "format_type", "compression_codec" ], "properties": { "format_type": { "title": "Format Type", "type": "string", "enum": [ "Avro" ], "default": "Avro", "order": 0 }, "compression_codec": { "title": "Compression Codec", "description": "The compression algorithm used to compress data. Default to no compression.", "type": "object", "oneOf": [ { "title": "No Compression", "required": [ "codec" ], "properties": { "codec": { "type": "string", "enum": [ "no compression" ], "default": "no compression" } } }, { "title": "Deflate", "required": [ "codec", "compression_level" ], "properties": { "codec": { "type": "string", "enum": [ "Deflate" ], "default": "Deflate" }, "compression_level": { "title": "Deflate Level", "description": "0: no compression & fastest, 9: best compression & slowest.", "type": "integer", "default": 0, "minimum": 0, "maximum": 9 } } }, { "title": "bzip2", "required": [ "codec" ], "properties": { "codec": { "type": "string", "enum": [ "bzip2" ], "default": "bzip2" } } }, { "title": "xz", "required": [ "codec", "compression_level" ], "properties": { "codec": { "type": "string", "enum": [ "xz" ], "default": "xz" }, "compression_level": { "title": "Compression Level", "description": "See here for details.", "type": "integer", "default": 6, "minimum": 0, "maximum": 9 } } }, { "title": "zstandard", "required": [ "codec", "compression_level" ], "properties": { "codec": { "type": "string", "enum": [ "zstandard" ], "default": "zstandard" }, "compression_level": { "title": "Compression Level", "description": "Negative levels are 'fast' modes akin to lz4 or snappy, levels above 9 are generally for archival purposes, and levels above 18 use a lot of memory.", "type": "integer", "default": 3, "minimum": -5, "maximum": 22 }, "include_checksum": { "title": "Include Checksum", "description": "If true, include a checksum with each data block.", "type": "boolean", "default": false } } }, { "title": "snappy", "required": [ "codec" ], "properties": { "codec": { "type": "string", "enum": [ "snappy" ], "default": "snappy" } } } ], "order": 1 } } }, { "title": "Parquet: Columnar Storage", "required": [ "format_type" ], "properties": { "format_type": { "title": "Format Type", "type": "string", "enum": [ "Parquet" ], "default": "Parquet" }, "compression_codec": { "title": "Compression Codec", "description": "The compression algorithm used to compress data pages.", "type": "string", "enum": [ "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "BROTLI", "LZ4", "ZSTD" ], "default": "UNCOMPRESSED" }, "block_size_mb": { "title": "Block Size (Row Group Size) (MB)", "description": "This is the size of a row group being buffered in memory. It limits the memory usage when writing. Larger values will improve the IO when reading, but consume more memory when writing. Default: 128 MB.", "type": "integer", "default": 128, "examples": [ 128 ] }, "max_padding_size_mb": { "title": "Max Padding Size (MB)", "description": "Maximum size allowed as padding to align row groups. This is also the minimum size of a row group. Default: 8 MB.", "type": "integer", "default": 8, "examples": [ 8 ] }, "page_size_kb": { "title": "Page Size (KB)", "description": "The page size is for compression. A block is composed of pages. A page is the smallest unit that must be read fully to access a single record. If this value is too small, the compression will deteriorate. Default: 1024 KB.", "type": "integer", "default": 1024, "examples": [ 1024 ] }, "dictionary_page_size_kb": { "title": "Dictionary Page Size (KB)", "description": "There is one dictionary page per column per row group when dictionary encoding is used. The dictionary page size works like the page size but for dictionary. Default: 1024 KB.", "type": "integer", "default": 1024, "examples": [ 1024 ] }, "dictionary_encoding": { "title": "Dictionary Encoding", "description": "Default: true.", "type": "boolean", "default": true } } } ], "order": 5 }, "s3_endpoint": { "title": "Endpoint", "type": "string", "default": "", "description": "Your S3 endpoint url. Read more here", "examples": [ "http://localhost:9000" ], "order": 6 }, "s3_path_format": { "title": "S3 Path Format", "description": "Format string on how data will be organized inside the S3 bucket directory. Read more here", "type": "string", "examples": [ "${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_" ], "order": 7 }, "file_name_pattern": { "type": "string", "description": "The pattern allows you to set the file-name format for the S3 staging file(s)", "title": "S3 Filename pattern", "examples": [ "{date}", "{date:yyyy_MM}", "{timestamp}", "{part_number}", "{sync_id}" ], "order": 8 }, "destinationType": { "title": "s3", "const": "s3", "enum": [ "s3" ], "order": 0, "type": "string" } } }, "destination-redis": { "title": "Redis Destination Spec", "type": "object", "required": [ "host", "username", "port", "cache_type", "destinationType" ], "properties": { "host": { "title": "Host", "description": "Redis host to connect to.", "type": "string", "examples": [ "localhost,127.0.0.1" ], "order": 1 }, "port": { "title": "Port", "description": "Port of Redis.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 6379, "order": 2 }, "username": { "title": "Username", "description": "Username associated with Redis.", "type": "string", "order": 3 }, "password": { "title": "Password", "description": "Password associated with Redis.", "type": "string", "airbyte_secret": true, "order": 4, "x-speakeasy-param-sensitive": true }, "ssl": { "title": "SSL Connection", "type": "boolean", "description": "Indicates whether SSL encryption protocol will be used to connect to Redis. It is recommended to use SSL connection if possible.", "default": false, "order": 5 }, "ssl_mode": { "title": "SSL Modes", "description": "SSL connection modes. \n
  • verify-full - This is the most secure mode. Always require encryption and verifies the identity of the source database server", "type": "object", "order": 6, "oneOf": [ { "title": "disable", "additionalProperties": false, "description": "Disable SSL.", "required": [ "mode" ], "properties": { "mode": { "type": "string", "const": "disable", "enum": [ "disable" ], "default": "disable", "order": 0 } } }, { "title": "verify-full", "additionalProperties": false, "description": "Verify-full SSL mode.", "required": [ "mode", "ca_certificate", "client_certificate", "client_key" ], "properties": { "mode": { "type": "string", "const": "verify-full", "enum": [ "verify-full" ], "default": "verify-full", "order": 0 }, "ca_certificate": { "type": "string", "title": "CA Certificate", "description": "CA certificate", "airbyte_secret": true, "multiline": true, "order": 1 }, "client_certificate": { "type": "string", "title": "Client Certificate", "description": "Client certificate", "airbyte_secret": true, "multiline": true, "order": 2 }, "client_key": { "type": "string", "title": "Client Key", "description": "Client key", "airbyte_secret": true, "multiline": true, "order": 3, "x-speakeasy-param-sensitive": true }, "client_key_password": { "type": "string", "title": "Client key password", "description": "Password for keystorage. If you do not add it - the password will be generated automatically.", "airbyte_secret": true, "order": 4, "x-speakeasy-param-sensitive": true } } } ] }, "cache_type": { "title": "Cache type", "type": "string", "default": "hash", "description": "Redis cache type to store data in.", "enum": [ "hash" ], "order": 7 }, "tunnel_method": { "type": "object", "title": "SSH Tunnel Method", "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", "oneOf": [ { "title": "No Tunnel", "required": [ "tunnel_method" ], "properties": { "tunnel_method": { "description": "No ssh tunnel needed to connect to database", "type": "string", "const": "NO_TUNNEL", "order": 0, "enum": [ "NO_TUNNEL" ] } } }, { "title": "SSH Key Authentication", "required": [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "ssh_key" ], "properties": { "tunnel_method": { "description": "Connect through a jump server tunnel host using username and ssh key", "type": "string", "const": "SSH_KEY_AUTH", "order": 0, "enum": [ "SSH_KEY_AUTH" ] }, "tunnel_host": { "title": "SSH Tunnel Jump Server Host", "description": "Hostname of the jump server host that allows inbound ssh tunnel.", "type": "string", "order": 1 }, "tunnel_port": { "title": "SSH Connection Port", "description": "Port on the proxy/jump server that accepts inbound ssh connections.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 22, "examples": [ "22" ], "order": 2 }, "tunnel_user": { "title": "SSH Login Username", "description": "OS-level username for logging into the jump server host.", "type": "string", "order": 3 }, "ssh_key": { "title": "SSH Private Key", "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", "type": "string", "airbyte_secret": true, "multiline": true, "order": 4, "x-speakeasy-param-sensitive": true } } }, { "title": "Password Authentication", "required": [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "tunnel_user_password" ], "properties": { "tunnel_method": { "description": "Connect through a jump server tunnel host using username and password authentication", "type": "string", "const": "SSH_PASSWORD_AUTH", "order": 0, "enum": [ "SSH_PASSWORD_AUTH" ] }, "tunnel_host": { "title": "SSH Tunnel Jump Server Host", "description": "Hostname of the jump server host that allows inbound ssh tunnel.", "type": "string", "order": 1 }, "tunnel_port": { "title": "SSH Connection Port", "description": "Port on the proxy/jump server that accepts inbound ssh connections.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 22, "examples": [ "22" ], "order": 2 }, "tunnel_user": { "title": "SSH Login Username", "description": "OS-level username for logging into the jump server host", "type": "string", "order": 3 }, "tunnel_user_password": { "title": "Password", "description": "OS-level password for logging into the jump server host", "type": "string", "airbyte_secret": true, "order": 4, "x-speakeasy-param-sensitive": true } } } ] }, "destinationType": { "title": "redis", "const": "redis", "enum": [ "redis" ], "order": 0, "type": "string" } } }, "destination-typesense": { "title": "Destination Typesense", "type": "object", "required": [ "api_key", "host", "destinationType" ], "properties": { "api_key": { "title": "API Key", "type": "string", "description": "Typesense API Key", "order": 0, "x-speakeasy-param-sensitive": true }, "host": { "title": "Host", "type": "string", "description": "Hostname of the Typesense instance without protocol.", "order": 1 }, "port": { "title": "Port", "type": "string", "description": "Port of the Typesense instance. Ex: 8108, 80, 443. Default is 443", "order": 2 }, "protocol": { "title": "Protocol", "type": "string", "description": "Protocol of the Typesense instance. Ex: http or https. Default is https", "order": 3 }, "batch_size": { "title": "Batch size", "type": "integer", "description": "How many documents should be imported together. Default 1000", "order": 4 }, "destinationType": { "title": "typesense", "const": "typesense", "enum": [ "typesense" ], "order": 0, "type": "string" } } }, "destination-bigquery": { "title": "BigQuery Destination Spec", "type": "object", "required": [ "project_id", "dataset_location", "dataset_id", "destinationType" ], "properties": { "project_id": { "type": "string", "description": "The GCP project ID for the project containing the target BigQuery dataset. Read more here.", "title": "Project ID", "group": "connection", "order": 0 }, "dataset_location": { "type": "string", "description": "The location of the dataset. Warning: Changes made after creation will not be applied. Read more here.", "title": "Dataset Location", "group": "connection", "order": 1, "enum": [ "US", "EU", "asia-east1", "asia-east2", "asia-northeast1", "asia-northeast2", "asia-northeast3", "asia-south1", "asia-south2", "asia-southeast1", "asia-southeast2", "australia-southeast1", "australia-southeast2", "europe-central1", "europe-central2", "europe-north1", "europe-southwest1", "europe-west1", "europe-west2", "europe-west3", "europe-west4", "europe-west6", "europe-west7", "europe-west8", "europe-west9", "europe-west12", "me-central1", "me-central2", "me-west1", "northamerica-northeast1", "northamerica-northeast2", "southamerica-east1", "southamerica-west1", "us-central1", "us-east1", "us-east2", "us-east3", "us-east4", "us-east5", "us-south1", "us-west1", "us-west2", "us-west3", "us-west4" ] }, "dataset_id": { "type": "string", "description": "The default BigQuery Dataset ID that tables are replicated to if the source does not specify a namespace. Read more here.", "title": "Default Dataset ID", "group": "connection", "order": 2 }, "loading_method": { "type": "object", "title": "Loading Method", "description": "The way data will be uploaded to BigQuery.", "display_type": "radio", "group": "connection", "order": 3, "oneOf": [ { "title": "GCS Staging", "description": "(recommended) Writes large batches of records to a file, uploads the file to GCS, then uses COPY INTO to load your data into BigQuery. Provides best-in-class speed, reliability and scalability. Read more about GCS Staging here.", "required": [ "method", "gcs_bucket_name", "gcs_bucket_path", "credential" ], "properties": { "method": { "type": "string", "const": "GCS Staging", "order": 0, "enum": [ "GCS Staging" ] }, "credential": { "title": "Credential", "description": "An HMAC key is a type of credential and can be associated with a service account or a user account in Cloud Storage. Read more here.", "type": "object", "order": 1, "oneOf": [ { "title": "HMAC key", "required": [ "credential_type", "hmac_key_access_id", "hmac_key_secret" ], "properties": { "credential_type": { "type": "string", "const": "HMAC_KEY", "order": 0, "enum": [ "HMAC_KEY" ] }, "hmac_key_access_id": { "type": "string", "description": "HMAC key access ID. When linked to a service account, this ID is 61 characters long; when linked to a user account, it is 24 characters long.", "title": "HMAC Key Access ID", "airbyte_secret": true, "examples": [ "1234567890abcdefghij1234" ], "order": 1, "x-speakeasy-param-sensitive": true }, "hmac_key_secret": { "type": "string", "description": "The corresponding secret for the access ID. It is a 40-character base-64 encoded string.", "title": "HMAC Key Secret", "airbyte_secret": true, "examples": [ "1234567890abcdefghij1234567890ABCDEFGHIJ" ], "order": 2, "x-speakeasy-param-sensitive": true } } } ] }, "gcs_bucket_name": { "title": "GCS Bucket Name", "type": "string", "description": "The name of the GCS bucket. Read more here.", "examples": [ "airbyte_sync" ], "order": 2 }, "gcs_bucket_path": { "title": "GCS Bucket Path", "description": "Directory under the GCS bucket where data will be written.", "type": "string", "examples": [ "data_sync/test" ], "order": 3 }, "keep_files_in_gcs-bucket": { "type": "string", "description": "This upload method is supposed to temporary store records in GCS bucket. By this select you can chose if these records should be removed from GCS when migration has finished. The default \"Delete all tmp files from GCS\" value is used if not set explicitly.", "title": "GCS Tmp Files Afterward Processing", "default": "Delete all tmp files from GCS", "enum": [ "Delete all tmp files from GCS", "Keep all tmp files in GCS" ], "order": 4 } } }, { "title": "Standard Inserts", "required": [ "method" ], "description": "(not recommended) Direct loading using SQL INSERT statements. This method is extremely inefficient and provided only for quick testing. In all other cases, you should use GCS staging.", "properties": { "method": { "type": "string", "const": "Standard", "enum": [ "Standard" ] } } } ] }, "credentials_json": { "type": "string", "description": "The contents of the JSON service account key. Check out the docs if you need help generating this key. Default credentials will be used if this field is left empty.", "title": "Service Account Key JSON (Required for cloud, optional for open-source)", "airbyte_secret": true, "group": "connection", "order": 4, "always_show": true }, "transformation_priority": { "type": "string", "description": "Interactive run type means that the query is executed as soon as possible, and these queries count towards concurrent rate limit and daily limit. Read more about interactive run type here. Batch queries are queued and started as soon as idle resources are available in the BigQuery shared resource pool, which usually occurs within a few minutes. Batch queries don’t count towards your concurrent rate limit. Read more about batch queries here. The default \"interactive\" value is used if not set explicitly.", "title": "Transformation Query Run Type", "default": "interactive", "enum": [ "interactive", "batch" ], "order": 5, "group": "advanced" }, "big_query_client_buffer_size_mb": { "title": "Google BigQuery Client Chunk Size", "description": "Google BigQuery client's chunk (buffer) size (MIN=1, MAX = 15) for each table. The size that will be written by a single RPC. Written data will be buffered and only flushed upon reaching this size or closing the channel. The default 15MB value is used if not set explicitly. Read more here.", "type": "integer", "minimum": 1, "maximum": 15, "default": 15, "examples": [ "15" ], "order": 6, "group": "advanced" }, "raw_data_dataset": { "type": "string", "description": "The dataset to write raw tables into (default: airbyte_internal)", "title": "Raw Table Dataset Name", "order": 7, "group": "advanced" }, "disable_type_dedupe": { "type": "boolean", "default": false, "description": "Disable Writing Final Tables. WARNING! The data format in _airbyte_data is likely stable but there are no guarantees that other metadata columns will remain the same in future versions", "title": "Disable Final Tables. (WARNING! Unstable option; Columns in raw table schema might change between versions)", "order": 8, "group": "advanced" }, "destinationType": { "title": "bigquery", "const": "bigquery", "enum": [ "bigquery" ], "order": 0, "type": "string" } }, "groups": [ { "id": "connection", "title": "Connection" }, { "id": "advanced", "title": "Advanced" } ] }, "destination-vertica": { "title": "Vertica Destination Spec", "type": "object", "required": [ "host", "port", "username", "database", "schema", "destinationType" ], "properties": { "host": { "title": "Host", "description": "Hostname of the database.", "type": "string", "order": 0 }, "port": { "title": "Port", "description": "Port of the database.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 5433, "examples": [ "5433" ], "order": 1 }, "database": { "title": "DB Name", "description": "Name of the database.", "type": "string", "order": 2 }, "username": { "title": "User", "description": "Username to use to access the database.", "type": "string", "order": 3 }, "password": { "title": "Password", "description": "Password associated with the username.", "type": "string", "airbyte_secret": true, "order": 4, "x-speakeasy-param-sensitive": true }, "jdbc_url_params": { "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "title": "JDBC URL Params", "type": "string", "order": 6 }, "schema": { "title": "Schema", "description": "Schema for vertica destination", "type": "string", "order": 7 }, "tunnel_method": { "type": "object", "title": "SSH Tunnel Method", "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", "oneOf": [ { "title": "No Tunnel", "required": [ "tunnel_method" ], "properties": { "tunnel_method": { "description": "No ssh tunnel needed to connect to database", "type": "string", "const": "NO_TUNNEL", "order": 0, "enum": [ "NO_TUNNEL" ] } } }, { "title": "SSH Key Authentication", "required": [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "ssh_key" ], "properties": { "tunnel_method": { "description": "Connect through a jump server tunnel host using username and ssh key", "type": "string", "const": "SSH_KEY_AUTH", "order": 0, "enum": [ "SSH_KEY_AUTH" ] }, "tunnel_host": { "title": "SSH Tunnel Jump Server Host", "description": "Hostname of the jump server host that allows inbound ssh tunnel.", "type": "string", "order": 1 }, "tunnel_port": { "title": "SSH Connection Port", "description": "Port on the proxy/jump server that accepts inbound ssh connections.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 22, "examples": [ "22" ], "order": 2 }, "tunnel_user": { "title": "SSH Login Username", "description": "OS-level username for logging into the jump server host.", "type": "string", "order": 3 }, "ssh_key": { "title": "SSH Private Key", "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", "type": "string", "airbyte_secret": true, "multiline": true, "order": 4, "x-speakeasy-param-sensitive": true } } }, { "title": "Password Authentication", "required": [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "tunnel_user_password" ], "properties": { "tunnel_method": { "description": "Connect through a jump server tunnel host using username and password authentication", "type": "string", "const": "SSH_PASSWORD_AUTH", "order": 0, "enum": [ "SSH_PASSWORD_AUTH" ] }, "tunnel_host": { "title": "SSH Tunnel Jump Server Host", "description": "Hostname of the jump server host that allows inbound ssh tunnel.", "type": "string", "order": 1 }, "tunnel_port": { "title": "SSH Connection Port", "description": "Port on the proxy/jump server that accepts inbound ssh connections.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 22, "examples": [ "22" ], "order": 2 }, "tunnel_user": { "title": "SSH Login Username", "description": "OS-level username for logging into the jump server host", "type": "string", "order": 3 }, "tunnel_user_password": { "title": "Password", "description": "OS-level password for logging into the jump server host", "type": "string", "airbyte_secret": true, "order": 4, "x-speakeasy-param-sensitive": true } } } ] }, "destinationType": { "title": "vertica", "const": "vertica", "enum": [ "vertica" ], "order": 0, "type": "string" } } }, "destination-elasticsearch": { "title": "Elasticsearch Connection Configuration", "type": "object", "required": [ "endpoint", "destinationType" ], "properties": { "endpoint": { "title": "Server Endpoint", "type": "string", "description": "The full url of the Elasticsearch server" }, "upsert": { "type": "boolean", "title": "Upsert Records", "description": "If a primary key identifier is defined in the source, an upsert will be performed using the primary key value as the elasticsearch doc id. Does not support composite primary keys.", "default": true }, "ca_certificate": { "type": "string", "title": "CA certificate", "description": "CA certificate", "airbyte_secret": true, "multiline": true }, "authenticationMethod": { "title": "Authentication Method", "type": "object", "description": "The type of authentication to be used", "oneOf": [ { "title": "Api Key/Secret", "additionalProperties": false, "description": "Use a api key and secret combination to authenticate", "required": [ "method", "apiKeyId", "apiKeySecret" ], "properties": { "method": { "type": "string", "const": "secret", "enum": [ "secret" ] }, "apiKeyId": { "title": "API Key ID", "description": "The Key ID to used when accessing an enterprise Elasticsearch instance.", "type": "string" }, "apiKeySecret": { "title": "API Key Secret", "description": "The secret associated with the API Key ID.", "type": "string", "airbyte_secret": true } } }, { "title": "Username/Password", "additionalProperties": false, "description": "Basic auth header with a username and password", "required": [ "method", "username", "password" ], "properties": { "method": { "type": "string", "const": "basic", "enum": [ "basic" ] }, "username": { "title": "Username", "description": "Basic auth username to access a secure Elasticsearch server", "type": "string" }, "password": { "title": "Password", "description": "Basic auth password to access a secure Elasticsearch server", "type": "string", "airbyte_secret": true, "x-speakeasy-param-sensitive": true } } } ] }, "destinationType": { "title": "elasticsearch", "const": "elasticsearch", "enum": [ "elasticsearch" ], "order": 0, "type": "string" } } }, "destination-azure-blob-storage": { "title": "AzureBlobStorage Destination Spec", "type": "object", "required": [ "azure_blob_storage_account_name", "azure_blob_storage_account_key", "format", "destinationType" ], "properties": { "azure_blob_storage_endpoint_domain_name": { "title": "Endpoint Domain Name", "type": "string", "default": "blob.core.windows.net", "description": "This is Azure Blob Storage endpoint domain name. Leave default value (or leave it empty if run container from command line) to use Microsoft native from example.", "examples": [ "blob.core.windows.net" ] }, "azure_blob_storage_container_name": { "title": "Azure blob storage container (Bucket) Name", "type": "string", "description": "The name of the Azure blob storage container. If not exists - will be created automatically. May be empty, then will be created automatically airbytecontainer+timestamp", "examples": [ "airbytetescontainername" ] }, "azure_blob_storage_account_name": { "title": "Azure Blob Storage account name", "type": "string", "description": "The account's name of the Azure Blob Storage.", "examples": [ "airbyte5storage" ] }, "azure_blob_storage_account_key": { "title": "Azure Blob Storage account key", "description": "The Azure blob storage account key.", "airbyte_secret": true, "type": "string", "examples": [ "Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd==" ], "x-speakeasy-param-sensitive": true }, "azure_blob_storage_output_buffer_size": { "title": "Azure Blob Storage output buffer size (Megabytes)", "type": "integer", "description": "The amount of megabytes to buffer for the output stream to Azure. This will impact memory footprint on workers, but may need adjustment for performance and appropriate block size in Azure.", "minimum": 1, "maximum": 2047, "default": 5, "examples": [ 5 ] }, "azure_blob_storage_spill_size": { "title": "Azure Blob Storage file spill size", "type": "integer", "description": "The amount of megabytes after which the connector should spill the records in a new blob object. Make sure to configure size greater than individual records. Enter 0 if not applicable", "default": 500, "examples": [ 500 ] }, "format": { "title": "Output Format", "type": "object", "description": "Output data format", "oneOf": [ { "title": "CSV: Comma-Separated Values", "required": [ "format_type", "flattening" ], "properties": { "format_type": { "type": "string", "const": "CSV", "enum": [ "CSV" ] }, "flattening": { "type": "string", "title": "Normalization (Flattening)", "description": "Whether the input json data should be normalized (flattened) in the output CSV. Please refer to docs for details.", "default": "No flattening", "enum": [ "No flattening", "Root level flattening" ] } } }, { "title": "JSON Lines: newline-delimited JSON", "required": [ "format_type" ], "properties": { "format_type": { "type": "string", "const": "JSONL", "enum": [ "JSONL" ] } } } ] }, "destinationType": { "title": "azure-blob-storage", "const": "azure-blob-storage", "enum": [ "azure-blob-storage" ], "order": 0, "type": "string" } } }, "destination-langchain": { "title": "Langchain Destination Config", "type": "object", "properties": { "processing": { "title": "ProcessingConfigModel", "type": "object", "properties": { "chunk_size": { "title": "Chunk size", "description": "Size of chunks in tokens to store in vector store (make sure it is not too big for the context if your LLM)", "maximum": 8191, "type": "integer" }, "chunk_overlap": { "title": "Chunk overlap", "description": "Size of overlap between chunks in tokens to store in vector store to better capture relevant context", "default": 0, "type": "integer" }, "text_fields": { "title": "Text fields to embed", "description": "List of fields in the record that should be used to calculate the embedding. All other fields are passed along as meta fields. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered text fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.", "always_show": true, "examples": [ "text", "user.name", "users.*.name" ], "type": "array", "items": { "type": "string" } } }, "required": [ "chunk_size", "text_fields" ], "group": "processing" }, "embedding": { "title": "Embedding", "description": "Embedding configuration", "group": "embedding", "type": "object", "oneOf": [ { "title": "OpenAI", "type": "object", "properties": { "mode": { "title": "Mode", "default": "openai", "const": "openai", "enum": [ "openai" ], "type": "string" }, "openai_key": { "title": "OpenAI API key", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true } }, "required": [ "openai_key" ], "description": "Use the OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." }, { "title": "Fake", "type": "object", "properties": { "mode": { "title": "Mode", "default": "fake", "const": "fake", "enum": [ "fake" ], "type": "string" } }, "description": "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs." } ] }, "indexing": { "title": "Indexing", "description": "Indexing configuration", "group": "indexing", "type": "object", "oneOf": [ { "title": "Pinecone", "type": "object", "properties": { "mode": { "title": "Mode", "default": "pinecone", "const": "pinecone", "enum": [ "pinecone" ], "type": "string" }, "pinecone_key": { "title": "Pinecone API key", "airbyte_secret": true, "type": "string", "x-speakeasy-param-sensitive": true }, "pinecone_environment": { "title": "Pinecone environment", "description": "Pinecone environment to use", "type": "string" }, "index": { "title": "Index", "description": "Pinecone index to use", "type": "string" } }, "required": [ "pinecone_key", "pinecone_environment", "index" ], "description": "Pinecone is a popular vector store that can be used to store and retrieve embeddings. It is a managed service and can also be queried from outside of langchain." }, { "title": "DocArrayHnswSearch", "type": "object", "properties": { "mode": { "title": "Mode", "default": "DocArrayHnswSearch", "const": "DocArrayHnswSearch", "enum": [ "DocArrayHnswSearch" ], "type": "string" }, "destination_path": { "title": "Destination Path", "description": "Path to the directory where hnswlib and meta data files will be written. The files will be placed inside that local mount. All files in the specified destination directory will be deleted on each run.", "examples": [ "/local/my_hnswlib_index" ], "type": "string" } }, "required": [ "destination_path" ], "description": "DocArrayHnswSearch is a lightweight Document Index implementation provided by Docarray that runs fully locally and is best suited for small- to medium-sized datasets. It stores vectors on disk in hnswlib, and stores all other data in SQLite." }, { "title": "Chroma (local persistance)", "type": "object", "properties": { "mode": { "title": "Mode", "default": "chroma_local", "const": "chroma_local", "enum": [ "chroma_local" ], "type": "string" }, "destination_path": { "title": "Destination Path", "description": "Path to the directory where chroma files will be written. The files will be placed inside that local mount.", "examples": [ "/local/my_chroma_db" ], "type": "string" }, "collection_name": { "title": "Collection Name", "description": "Name of the collection to use.", "default": "langchain", "type": "string" } }, "required": [ "destination_path" ], "description": "Chroma is a popular vector store that can be used to store and retrieve embeddings. It will build its index in memory and persist it to disk by the end of the sync." } ] }, "destinationType": { "title": "langchain", "const": "langchain", "enum": [ "langchain" ], "order": 0, "type": "string" } }, "required": [ "processing", "embedding", "indexing", "destinationType" ], "groups": [ { "id": "processing", "title": "Processing" }, { "id": "embedding", "title": "Embedding" }, { "id": "indexing", "title": "Indexing" } ] }, "destination-cumulio": { "title": "Destination Cumulio", "type": "object", "required": [ "api_host", "api_key", "api_token", "destinationType" ], "properties": { "api_host": { "title": "Cumul.io API Host URL", "description": "URL of the Cumul.io API (e.g. 'https://api.cumul.io', 'https://api.us.cumul.io', or VPC-specific API url). Defaults to 'https://api.cumul.io'.", "default": "https://api.cumul.io", "type": "string", "order": 0 }, "api_key": { "title": "Cumul.io API Key", "description": "An API key generated in Cumul.io's platform (can be generated here: https://app.cumul.io/start/profile/integration).", "type": "string", "airbyte_secret": true, "order": 1, "x-speakeasy-param-sensitive": true }, "api_token": { "title": "Cumul.io API Token", "description": "The corresponding API token generated in Cumul.io's platform (can be generated here: https://app.cumul.io/start/profile/integration).", "type": "string", "airbyte_secret": true, "order": 2, "x-speakeasy-param-sensitive": true }, "destinationType": { "title": "cumulio", "const": "cumulio", "enum": [ "cumulio" ], "order": 0, "type": "string" } } }, "destination-postgres": { "title": "Postgres Destination Spec", "type": "object", "required": [ "host", "port", "username", "database", "schema", "destinationType" ], "properties": { "host": { "title": "Host", "description": "Hostname of the database.", "type": "string", "order": 0 }, "port": { "title": "Port", "description": "Port of the database.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 5432, "examples": [ "5432" ], "order": 1 }, "database": { "title": "DB Name", "description": "Name of the database.", "type": "string", "order": 2 }, "schema": { "title": "Default Schema", "description": "The default schema tables are written to if the source does not specify a namespace. The usual value for this field is \"public\".", "type": "string", "examples": [ "public" ], "default": "public", "order": 3 }, "username": { "title": "User", "description": "Username to use to access the database.", "type": "string", "order": 4 }, "password": { "title": "Password", "description": "Password associated with the username.", "type": "string", "airbyte_secret": true, "order": 5, "x-speakeasy-param-sensitive": true }, "ssl_mode": { "title": "SSL modes", "description": "SSL connection modes. \n disable - Chose this mode to disable encryption of communication between Airbyte and destination database\n allow - Chose this mode to enable encryption only when required by the source database\n prefer - Chose this mode to allow unencrypted connection only if the source database does not support encryption\n require - Chose this mode to always require encryption. If the source database server does not support encryption, connection will fail\n verify-ca - Chose this mode to always require encryption and to verify that the source database server has a valid SSL certificate\n verify-full - This is the most secure mode. Chose this mode to always require encryption and to verify the identity of the source database server\n See more information - in the docs.", "type": "object", "order": 7, "oneOf": [ { "title": "disable", "additionalProperties": false, "description": "Disable SSL.", "required": [ "mode" ], "properties": { "mode": { "type": "string", "const": "disable", "enum": [ "disable" ], "default": "disable", "order": 0 } } }, { "title": "allow", "additionalProperties": false, "description": "Allow SSL mode.", "required": [ "mode" ], "properties": { "mode": { "type": "string", "const": "allow", "enum": [ "allow" ], "default": "allow", "order": 0 } } }, { "title": "prefer", "additionalProperties": false, "description": "Prefer SSL mode.", "required": [ "mode" ], "properties": { "mode": { "type": "string", "const": "prefer", "enum": [ "prefer" ], "default": "prefer", "order": 0 } } }, { "title": "require", "additionalProperties": false, "description": "Require SSL mode.", "required": [ "mode" ], "properties": { "mode": { "type": "string", "const": "require", "enum": [ "require" ], "default": "require", "order": 0 } } }, { "title": "verify-ca", "additionalProperties": false, "description": "Verify-ca SSL mode.", "required": [ "mode", "ca_certificate" ], "properties": { "mode": { "type": "string", "const": "verify-ca", "enum": [ "verify-ca" ], "default": "verify-ca", "order": 0 }, "ca_certificate": { "type": "string", "title": "CA certificate", "description": "CA certificate", "airbyte_secret": true, "multiline": true, "order": 1 }, "client_key_password": { "type": "string", "title": "Client key password", "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", "airbyte_secret": true, "order": 4, "x-speakeasy-param-sensitive": true } } }, { "title": "verify-full", "additionalProperties": false, "description": "Verify-full SSL mode.", "required": [ "mode", "ca_certificate", "client_certificate", "client_key" ], "properties": { "mode": { "type": "string", "const": "verify-full", "enum": [ "verify-full" ], "default": "verify-full", "order": 0 }, "ca_certificate": { "type": "string", "title": "CA certificate", "description": "CA certificate", "airbyte_secret": true, "multiline": true, "order": 1 }, "client_certificate": { "type": "string", "title": "Client certificate", "description": "Client certificate", "airbyte_secret": true, "multiline": true, "order": 2 }, "client_key": { "type": "string", "title": "Client key", "description": "Client key", "airbyte_secret": true, "multiline": true, "order": 3, "x-speakeasy-param-sensitive": true }, "client_key_password": { "type": "string", "title": "Client key password", "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", "airbyte_secret": true, "order": 4, "x-speakeasy-param-sensitive": true } } } ] }, "jdbc_url_params": { "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "title": "JDBC URL Params", "type": "string", "order": 8 }, "raw_data_schema": { "type": "string", "description": "The schema to write raw tables into", "title": "Raw table schema (defaults to airbyte_internal)", "order": 9 }, "disable_type_dedupe": { "type": "boolean", "default": false, "description": "Disable Writing Final Tables. WARNING! The data format in _airbyte_data is likely stable but there are no guarantees that other metadata columns will remain the same in future versions", "title": "Disable Final Tables. (WARNING! Unstable option; Columns in raw table schema might change between versions)", "order": 10 }, "tunnel_method": { "type": "object", "title": "SSH Tunnel Method", "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", "oneOf": [ { "title": "No Tunnel", "required": [ "tunnel_method" ], "properties": { "tunnel_method": { "description": "No ssh tunnel needed to connect to database", "type": "string", "const": "NO_TUNNEL", "order": 0, "enum": [ "NO_TUNNEL" ] } } }, { "title": "SSH Key Authentication", "required": [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "ssh_key" ], "properties": { "tunnel_method": { "description": "Connect through a jump server tunnel host using username and ssh key", "type": "string", "const": "SSH_KEY_AUTH", "order": 0, "enum": [ "SSH_KEY_AUTH" ] }, "tunnel_host": { "title": "SSH Tunnel Jump Server Host", "description": "Hostname of the jump server host that allows inbound ssh tunnel.", "type": "string", "order": 1 }, "tunnel_port": { "title": "SSH Connection Port", "description": "Port on the proxy/jump server that accepts inbound ssh connections.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 22, "examples": [ "22" ], "order": 2 }, "tunnel_user": { "title": "SSH Login Username", "description": "OS-level username for logging into the jump server host.", "type": "string", "order": 3 }, "ssh_key": { "title": "SSH Private Key", "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", "type": "string", "airbyte_secret": true, "multiline": true, "order": 4, "x-speakeasy-param-sensitive": true } } }, { "title": "Password Authentication", "required": [ "tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "tunnel_user_password" ], "properties": { "tunnel_method": { "description": "Connect through a jump server tunnel host using username and password authentication", "type": "string", "const": "SSH_PASSWORD_AUTH", "order": 0, "enum": [ "SSH_PASSWORD_AUTH" ] }, "tunnel_host": { "title": "SSH Tunnel Jump Server Host", "description": "Hostname of the jump server host that allows inbound ssh tunnel.", "type": "string", "order": 1 }, "tunnel_port": { "title": "SSH Connection Port", "description": "Port on the proxy/jump server that accepts inbound ssh connections.", "type": "integer", "minimum": 0, "maximum": 65536, "default": 22, "examples": [ "22" ], "order": 2 }, "tunnel_user": { "title": "SSH Login Username", "description": "OS-level username for logging into the jump server host", "type": "string", "order": 3 }, "tunnel_user_password": { "title": "Password", "description": "OS-level password for logging into the jump server host", "type": "string", "airbyte_secret": true, "order": 4, "x-speakeasy-param-sensitive": true } } } ] }, "destinationType": { "title": "postgres", "const": "postgres", "enum": [ "postgres" ], "order": 0, "type": "string" } } }, "DestinationConfiguration": { "description": "The values required to configure the destination.", "example": { "user": "charles" }, "oneOf": [ { "title": "destination-google-sheets", "$ref": "#/components/schemas/destination-google-sheets" }, { "title": "destination-astra", "$ref": "#/components/schemas/destination-astra" }, { "title": "destination-aws-datalake", "$ref": "#/components/schemas/destination-aws-datalake" }, { "title": "destination-azure-blob-storage", "$ref": "#/components/schemas/destination-azure-blob-storage" }, { "title": "destination-bigquery", "$ref": "#/components/schemas/destination-bigquery" }, { "title": "destination-clickhouse", "$ref": "#/components/schemas/destination-clickhouse" }, { "title": "destination-convex", "$ref": "#/components/schemas/destination-convex" }, { "title": "destination-cumulio", "$ref": "#/components/schemas/destination-cumulio" }, { "title": "destination-databend", "$ref": "#/components/schemas/destination-databend" }, { "title": "destination-databricks", "$ref": "#/components/schemas/destination-databricks" }, { "title": "destination-dev-null", "$ref": "#/components/schemas/destination-dev-null" }, { "title": "destination-duckdb", "$ref": "#/components/schemas/destination-duckdb" }, { "title": "destination-dynamodb", "$ref": "#/components/schemas/destination-dynamodb" }, { "title": "destination-elasticsearch", "$ref": "#/components/schemas/destination-elasticsearch" }, { "title": "destination-firebolt", "$ref": "#/components/schemas/destination-firebolt" }, { "title": "destination-firestore", "$ref": "#/components/schemas/destination-firestore" }, { "title": "destination-gcs", "$ref": "#/components/schemas/destination-gcs" }, { "title": "destination-keen", "$ref": "#/components/schemas/destination-keen" }, { "title": "destination-kinesis", "$ref": "#/components/schemas/destination-kinesis" }, { "title": "destination-langchain", "$ref": "#/components/schemas/destination-langchain" }, { "title": "destination-milvus", "$ref": "#/components/schemas/destination-milvus" }, { "title": "destination-mongodb", "$ref": "#/components/schemas/destination-mongodb" }, { "title": "destination-mssql", "$ref": "#/components/schemas/destination-mssql" }, { "title": "destination-mysql", "$ref": "#/components/schemas/destination-mysql" }, { "title": "destination-oracle", "$ref": "#/components/schemas/destination-oracle" }, { "title": "destination-pinecone", "$ref": "#/components/schemas/destination-pinecone" }, { "title": "destination-postgres", "$ref": "#/components/schemas/destination-postgres" }, { "title": "destination-pubsub", "$ref": "#/components/schemas/destination-pubsub" }, { "title": "destination-qdrant", "$ref": "#/components/schemas/destination-qdrant" }, { "title": "destination-redis", "$ref": "#/components/schemas/destination-redis" }, { "title": "destination-redshift", "$ref": "#/components/schemas/destination-redshift" }, { "title": "destination-s3", "$ref": "#/components/schemas/destination-s3" }, { "title": "destination-s3-glue", "$ref": "#/components/schemas/destination-s3-glue" }, { "title": "destination-sftp-json", "$ref": "#/components/schemas/destination-sftp-json" }, { "title": "destination-snowflake", "$ref": "#/components/schemas/destination-snowflake" }, { "title": "destination-teradata", "$ref": "#/components/schemas/destination-teradata" }, { "title": "destination-timeplus", "$ref": "#/components/schemas/destination-timeplus" }, { "title": "destination-typesense", "$ref": "#/components/schemas/destination-typesense" }, { "title": "destination-vectara", "$ref": "#/components/schemas/destination-vectara" }, { "title": "destination-vertica", "$ref": "#/components/schemas/destination-vertica" }, { "title": "destination-weaviate", "$ref": "#/components/schemas/destination-weaviate" }, { "title": "destination-xata", "$ref": "#/components/schemas/destination-xata" } ] } }, "securitySchemes": { "bearerAuth": { "type": "http", "scheme": "bearer", "bearerFormat": "JWT" } } }, "security": [ { "bearerAuth": [] } ], "x-readme": { "explorer-enabled": true, "proxy-enabled": true } } ```