diff --git a/clojure/Makefile b/clojure/Makefile index 2bc77252b..9c917c587 100644 --- a/clojure/Makefile +++ b/clojure/Makefile @@ -49,6 +49,7 @@ test-unit: -n pgloader.cast-test \ -n pgloader.batch-test \ -n pgloader.ddl-test \ + -n pgloader.ddl.citus-test \ -n pgloader.load-file.parser-test \ -n pgloader.transforms-test \ -n pgloader.pg-service-test \ diff --git a/clojure/src/pgloader/ddl/citus.clj b/clojure/src/pgloader/ddl/citus.clj index b5ee70697..5192dbe3b 100644 --- a/clojure/src/pgloader/ddl/citus.clj +++ b/clojure/src/pgloader/ddl/citus.clj @@ -76,7 +76,9 @@ own-cols (mapv :column-name (:columns table-entry)) cast-text? (= :pgsql source-type) fmt-col (fn [tbl col] - (str tbl "." col (when cast-text? "::text"))) + (if cast-text? + (str "CAST(" tbl "." col " AS text)") + (str tbl "." col))) col-list (str/join ", " (concat [(fmt-col from-table-name dist-col-name)] (map #(fmt-col tname %) own-cols)))] diff --git a/clojure/src/pgloader/source/pgsql.clj b/clojure/src/pgloader/source/pgsql.clj index 616395464..6e94fb7ea 100644 --- a/clojure/src/pgloader/source/pgsql.clj +++ b/clojure/src/pgloader/source/pgsql.clj @@ -201,7 +201,7 @@ source-schema (or (:source-schema table-spec-entry) schema) base-sql (or citus-read-sql (let [col-names (mapv :column-name columns) - col-list (str/join ", " (map #(str "\"" % "\"") col-names))] + col-list (str/join ", " (map #(str "CAST(\"" % "\" AS text)") col-names))] (str "SELECT " col-list " FROM \"" source-schema "\".\"" table-name "\""))) sql (if (and ctid-lo ctid-hi) (str base-sql " " (first (ctid-where-sqlvec {:lo (str "'(" ctid-lo ",0)'::tid") diff --git a/clojure/test/pgloader/ddl/citus_test.clj b/clojure/test/pgloader/ddl/citus_test.clj index dc8671669..5220c9035 100644 --- a/clojure/test/pgloader/ddl/citus_test.clj +++ b/clojure/test/pgloader/ddl/citus_test.clj @@ -88,14 +88,16 @@ (citus/augment-catalog broken-catalog rules)))))) (deftest test-augment-catalog-pg-source-type - (testing "PG source type adds ::text casts to the read SQL" + (testing "PG source type wraps columns in CAST(... AS text) in the read SQL" (let [rules [{:type :distributed-from :table "splits" :using "biller_id" :from ["receivable_accounts"]}] aug (citus/augment-catalog sample-catalog rules :source-type :pgsql) entry (first (filter #(= "splits" (:table-name %)) aug)) sql (:citus-read-sql entry)] (is (string? sql)) - (is (re-find #"::text" sql))))) + (is (re-find #"CAST\(" sql)) + (is (re-find #"AS text\)" sql)) + (is (not (re-find #"::text" sql)))))) (deftest test-augment-catalog-from-chain (testing "Two-hop FROM chain traverses correctly" diff --git a/src/sources/pgsql/pgsql.lisp b/src/sources/pgsql/pgsql.lisp index 684c2070d..2a84c51bf 100644 --- a/src/sources/pgsql/pgsql.lisp +++ b/src/sources/pgsql/pgsql.lisp @@ -40,7 +40,7 @@ (let* ((cols (mapcar #'column-name (fields pgsql))) (sql (format nil - "SELECT ~{~s::text~^, ~} FROM ~s.~s" + "SELECT ~{CAST(~s AS text)~^, ~} FROM ~s.~s" cols (schema-source-name (table-schema (source pgsql))) (table-source-name (source pgsql))))) diff --git a/src/utils/citus.lisp b/src/utils/citus.lisp index b846cdb03..017d4d988 100644 --- a/src/utils/citus.lisp +++ b/src/utils/citus.lisp @@ -368,7 +368,7 @@ (format-citus-join-clause source-table (table-citus-rule target-table)))) (format nil - "SELECT ~{~a::text~^, ~} FROM ~s.~s ~a" + "SELECT ~{CAST(~a AS text)~^, ~} FROM ~s.~s ~a" cols (schema-source-name (table-schema source-table)) (table-source-name source-table) diff --git a/test/citus/README.md b/test/citus/README.md index 499ecd66f..47c621983 100644 --- a/test/citus/README.md +++ b/test/citus/README.md @@ -28,15 +28,15 @@ model, and finally the data is backfilled automatically in the target table thanks to generating queries like the following: ~~~ -SELECT "campaigns".company_id::text, - "impressions".id::text, - "impressions".ad_id::text, - "impressions".seen_at::text, - "impressions".site_url::text, - "impressions".cost_per_impression_usd::text, - "impressions".user_ip::text, - "impressions".user_data::text - FROM "public"."impressions" +SELECT CAST(campaigns.company_id AS text), + CAST(impressions.id AS text), + CAST(impressions.ad_id AS text), + CAST(impressions.seen_at AS text), + CAST(impressions.site_url AS text), + CAST(impressions.cost_per_impression_usd AS text), + CAST(impressions.user_ip AS text), + CAST(impressions.user_data AS text) + FROM "public"."impressions" JOIN "public"."ads" ON impressions.ad_id = ads.id JOIN "public"."campaigns" ON ads.campaign_id = campaigns.id ~~~