From 3913ba976fd4fc9c4895750c2341a7e51edca8cc Mon Sep 17 00:00:00 2001 From: luozihao <1165977584@qq.com> Date: Mon, 21 Jun 2021 10:31:30 +0800 Subject: [PATCH 1/3] Fixed the bug that partition tables cannot be migrated. --- src/package.lisp | 13 ++ src/pgsql/pgsql-ddl.lisp | 116 +++++++++++++++++- src/sources/common/api.lisp | 3 + src/sources/mysql/mysql-schema.lisp | 22 ++++ src/sources/mysql/mysql.lisp | 5 +- src/sources/mysql/sql/list-all-partitions.sql | 8 ++ src/utils/catalog.lisp | 17 ++- 7 files changed, 181 insertions(+), 3 deletions(-) create mode 100644 src/sources/mysql/sql/list-all-partitions.sql diff --git a/src/package.lisp b/src/package.lisp index 65feb0e..ef7cd73 100644 --- a/src/package.lisp +++ b/src/package.lisp @@ -58,6 +58,7 @@ #:fkey #:trigger #:procedure + #:partition #:cast ; generic function for sources @@ -74,6 +75,7 @@ #:make-fkey #:make-trigger #:make-procedure + #:make-partition #:catalog-name #:catalog-schema-list @@ -102,6 +104,7 @@ #:table-fkey-list #:table-trigger-list #:table-citus-rule + #:table-partition-list #:matview-name #:matview-source-name @@ -142,6 +145,13 @@ #:index-filter #:index-fk-deps + #:partition-name + #:partition-method + #:partition-expression + #:partition-schema + #:partition-description + #:partition-submethod + #:fkey-name #:fkey-oid #:fkey-foreign-table @@ -189,6 +199,8 @@ #:add-index #:find-index #:maybe-add-index + #:add-partition + #:find-partition #:add-fkey #:find-fkey #:maybe-add-fkey @@ -570,6 +582,7 @@ #:fetch-columns #:fetch-indexes #:fetch-foreign-keys + #:fetch-partitions #:fetch-comments #:get-column-sql-expression #:get-column-list diff --git a/src/pgsql/pgsql-ddl.lisp b/src/pgsql/pgsql-ddl.lisp index 03e8962..b3b6d54 100644 --- a/src/pgsql/pgsql-ddl.lisp +++ b/src/pgsql/pgsql-ddl.lisp @@ -56,6 +56,16 @@ (extension-name extension) cascade)) +;;; +;;; My Spilit Function +;;; +(defun string-split (str delim &optional (start 0)) + (let ((p1 (position delim str :start start :test #'char/=))) + (if p1 + (let ((p2 (position delim str :start p1))) + (cons (subseq str p1 p2) + (if p2 (string-split str delim p2) nil))) + nil))) ;;; @@ -91,7 +101,111 @@ (format s "~%WITH (~{~a = '~a'~^,~% ~})" (alexandria:alist-plist (table-storage-parameter-list table)))) - + (defparameter white-list (list + "integer" "bigint" "character varying" "text" "character" "numeric" "date" + "time without time zone" "timestamp without time zone" "time" "timestamp" + "bpchar" "nchar" "decimal")) + (defparameter first-part (if (table-partition-list table) (first (table-partition-list table)) NIL)) + (defparameter is-skip NIL) + (when (and first-part (partition-method first-part)) + (defparameter max-key-count 4) + (defparameter key-count 0) + (defparameter column-count (length (table-column-list table))) + (defparameter partition-column-list (string-split (remove #\` (partition-expression first-part)) #\,)) + (setf key-count (length partition-column-list)) + (when (or + (> key-count max-key-count) + (and (> key-count 1) (string= "RANGE COLUMNS" (partition-method first-part)))) + (log-message :warning + "~a.~a's partition key num(~d) exceed max value(~d), create as normal table" + (schema-name (partition-schema first-part)) + (table-name table) key-count max-key-count) + (setf is-skip 1) + ) + (when (and + (not is-skip) + (or (string= (partition-method first-part) "KEY") + (string= (partition-method first-part) "LINEAR KEY"))) + (setf is-skip 1) + (loop + :for partition-key-name + :in partition-column-list + :always is-skip + :do (progn + (loop + :for column + :in (table-column-list table) + :always is-skip + :do (progn + (when (string= (column-name column) partition-key-name) + (loop + :for white-type + :in white-list + :always is-skip + :do (progn + (when (string= (column-type-name column) white-type) + (setf is-skip NIL))))))))) + ) + (defparameter statement "") + (unless is-skip + (setf statement (concatenate 'string statement " PARTITION BY ")) + (cond + ((or (string= (partition-method first-part) "RANGE") (string= (partition-method first-part) "RANGE COLUMNS")) + (setf statement (concatenate 'string statement "RANGE"))) + ((or (string= (partition-method first-part) "LIST") (string= (partition-method first-part) "LIST COLUMNS")) + (setf statement (concatenate 'string statement "LIST"))) + ((or (string= (partition-method first-part) "HASH") (string= (partition-method first-part) "LINEAR HASH")) + (setf statement (concatenate 'string statement "HASH"))) + ((or (string= (partition-method first-part) "KEY") (string= (partition-method first-part) "LINEAR KEY")) + (setf statement (concatenate 'string statement "HASH"))) + (t + (progn + (setf is-skip 1) + (log-message :warning "Unknown partition type") + (log-message :warning "Unknown partition type: ~s, create this table(~s.~s) as non-part table" + (partition-method first-part) (partition-schema partition) (table-name table)))) + )) + (unless is-skip + (setf statement + (concatenate 'string statement (format nil "(~a) (" (remove #\` (partition-expression first-part))))) + ) + + (loop + :for partition + :in (table-partition-list table) + :never is-skip + :do (progn + (cond + ((or (string= (partition-method first-part) "RANGE") (string= (partition-method first-part) "RANGE COLUMNS")) + (setf statement + (concatenate 'string statement + (format nil " partition ~a values less than(~a) " + (remove #\` (partition-name partition)) (partition-description partition))))) + ((or (string= (partition-method first-part) "LIST") (string= (partition-method first-part) "LIST COLUMNS")) + (setf statement + (concatenate 'string statement + (format nil " partition ~a values(~a) " + (remove #\` (partition-name partition)) (partition-description partition))))) + ((or (string= (partition-method first-part) "HASH") (string= (partition-method first-part) "LINEAR HASH")) + (setf statement + (concatenate 'string statement + (format nil " partition ~a " + (remove #\` (partition-name partition)))))) + ((or (string= (partition-method first-part) "KEY") (string= (partition-method first-part) "LINEAR KEY")) + (setf statement + (concatenate 'string statement + (format nil " partition ~a " + (remove #\` (partition-name partition)))))) + (t + (progn + (setf is-skip 1) + (log-message :warning "Unknown partition type") + (log-message :warning "Unknown partition type: ~s, create this table(~s.~s) as non-part table" + (partition-method first-part) (partition-schema partition) (table-name table))))))) + (unless is-skip + (format s "~a)" statement) + ) + ) (when (table-tablespace table) (format s "~%TABLESPACE ~a" (table-tablespace table))) diff --git a/src/sources/common/api.lisp b/src/sources/common/api.lisp index 51f45a1..bc87609 100644 --- a/src/sources/common/api.lisp +++ b/src/sources/common/api.lisp @@ -147,6 +147,9 @@ (:documentation "Get the list of schema, tables and columns from the source database.")) +(defgeneric fetch-partitions (catalog db-copy &key including excluding) + (:documentation "Get the list of partitions from the source database.")) + (defgeneric fetch-indexes (catalog db-copy &key including excluding) (:documentation "Get the list of indexes from the source database.")) diff --git a/src/sources/mysql/mysql-schema.lisp b/src/sources/mysql/mysql-schema.lisp index 296963d..c457934 100644 --- a/src/sources/mysql/mysql-schema.lisp +++ b/src/sources/mysql/mysql-schema.lisp @@ -112,6 +112,28 @@ :finally (return schema))) +;;; +;;; MySQL Partitions +;;; +(defmethod fetch-partitions ((schema schema) (mysql copy-mysql) + &key including excluding) + "Get the list of MySQL partition per table." + (loop + :for (table-name submethod name method expression description) + :in (mysql-query (sql "/mysql/list-all-partitions.sql" + (db-name *connection*))) + :do (let* ((table (find-table schema table-name)) + (partition + (make-partition :name name ; further processing is needed + :schema schema + :submethod submethod + :method method + :expression expression + :description description))) + (add-partition table partition)) + :finally + (return schema))) + ;;; ;;; MySQL Foreign Keys ;;; diff --git a/src/sources/mysql/mysql.lisp b/src/sources/mysql/mysql.lisp index 9b2826b..6d6f804 100644 --- a/src/sources/mysql/mysql.lisp +++ b/src/sources/mysql/mysql.lisp @@ -164,7 +164,10 @@ Illegal ~a character starting at position ~a~@[: ~a~].~%" (fetch-columns schema mysql :including including :excluding excluding) - + ;; fetch partition metadata + (fetch-partitions schema mysql + :including including + :excluding excluding) ;; fetch view (and their columns) metadata, covering comments too (let* ((view-names (unless (eq :all materialize-views) (mapcar #'matview-source-name materialize-views))) diff --git a/src/sources/mysql/sql/list-all-partitions.sql b/src/sources/mysql/sql/list-all-partitions.sql new file mode 100644 index 0000000..2a64768 --- /dev/null +++ b/src/sources/mysql/sql/list-all-partitions.sql @@ -0,0 +1,8 @@ + SELECT DISTINCT table_name, subpartition_method, partition_name, + partition_method, + partition_expression, + partition_description + FROM information_schema.partitions + WHERE table_schema='~a' +GROUP BY table_name +ORDER BY partition_ordinal_position; \ No newline at end of file diff --git a/src/utils/catalog.lisp b/src/utils/catalog.lisp index ccbb736..1fee60f 100644 --- a/src/utils/catalog.lisp +++ b/src/utils/catalog.lisp @@ -52,7 +52,7 @@ ;; field is for SOURCE ;; column is for TARGET ;; citus is an extra slot for citus support - field-list column-list index-list fkey-list trigger-list citus-rule) + field-list column-list index-list fkey-list trigger-list citus-rule partition-list) (defstruct matview source-name name schema definition) @@ -84,6 +84,8 @@ name oid table columns pkey foreign-table foreign-columns condef update-rule delete-rule match-rule deferrable initially-deferred) +(defstruct partition submethod name method expression description schema) + ;;; ;;; An index, that might be underlying a e.g. UNIQUE constraint conname, in ;;; which case we need to use condef to build the index again from its @@ -117,6 +119,7 @@ (defgeneric add-index (object index &key)) (defgeneric add-fkey (object fkey &key)) (defgeneric add-comment (object comment &key)) +(defgeneric add-partition (object partition &key)) (defgeneric extension-list (object &key) (:documentation "Return the list of extensions found in OBJECT.")) @@ -147,6 +150,10 @@ (:documentation "Find an index by INDEX-NAME in a table OBJECT and return the index")) +(defgeneric find-partition (object partition-name &key key test) + (:documentation + "Find a partition by PARTITION-NAME in a table OBJECT and return the partition")) + (defgeneric find-fkey (object fkey-name &key key test) (:documentation "Find a foreign key by FKEY-NAME in a table OBJECT and return the fkey")) @@ -387,6 +394,14 @@ ;;; fkey queries return a row per index|fkey column rather than per ;;; index|fkey. Hence this extra API: ;;; +(defmethod add-partition ((table table) partition &key) + "Add PARTITION to TABLE and return the TABLE." + (push-to-end partition (table-partition-list table))) + +(defmethod find-partition ((table table) partition-name &key key (test #'string=)) + "Find PARTITION-NAME in TABLE and return the PARTITION object of this name." + (find partition-name (table-partition-list table) :key key :test test)) + (defmethod add-index ((table table) index &key) "Add INDEX to TABLE and return the TABLE." (push-to-end index (table-index-list table))) -- Gitee From 0f86956e1c301c3211e5b8565ffb17508ec15747 Mon Sep 17 00:00:00 2001 From: luozihao <1165977584@qq.com> Date: Mon, 21 Jun 2021 10:32:12 +0800 Subject: [PATCH 2/3] Add the warning message about subpartition --- src/pgsql/pgsql-ddl.lisp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pgsql/pgsql-ddl.lisp b/src/pgsql/pgsql-ddl.lisp index b3b6d54..698e410 100644 --- a/src/pgsql/pgsql-ddl.lisp +++ b/src/pgsql/pgsql-ddl.lisp @@ -108,6 +108,10 @@ (defparameter first-part (if (table-partition-list table) (first (table-partition-list table)) NIL)) (defparameter is-skip NIL) (when (and first-part (partition-method first-part)) + (when (partition-submethod first-part) + (log-message :warning + "~a.~a is a composite partition table, ignore subpartition" + (schema-name (partition-schema first-part)) (table-name table))) (defparameter max-key-count 4) (defparameter key-count 0) (defparameter column-count (length (table-column-list table))) -- Gitee From 2dcf756dd921002e90d8eeb898d099707e358983 Mon Sep 17 00:00:00 2001 From: luozihao <1165977584@qq.com> Date: Mon, 21 Jun 2021 11:34:22 +0800 Subject: [PATCH 3/3] Optimize the code structure and add appropriate comments --- src/pgsql/pgsql-ddl.lisp | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/src/pgsql/pgsql-ddl.lisp b/src/pgsql/pgsql-ddl.lisp index 698e410..aab548a 100644 --- a/src/pgsql/pgsql-ddl.lisp +++ b/src/pgsql/pgsql-ddl.lisp @@ -105,18 +105,26 @@ "integer" "bigint" "character varying" "text" "character" "numeric" "date" "time without time zone" "timestamp without time zone" "time" "timestamp" "bpchar" "nchar" "decimal")) + (defparameter first-part (if (table-partition-list table) (first (table-partition-list table)) NIL)) (defparameter is-skip NIL) + (when (and first-part (partition-method first-part)) + ;; If the current partition contains a SubPartition, + ;; treat it as a normal Partition and issue a warning (when (partition-submethod first-part) (log-message :warning "~a.~a is a composite partition table, ignore subpartition" (schema-name (partition-schema first-part)) (table-name table))) + (defparameter max-key-count 4) (defparameter key-count 0) (defparameter column-count (length (table-column-list table))) (defparameter partition-column-list (string-split (remove #\` (partition-expression first-part)) #\,)) (setf key-count (length partition-column-list)) + + + ;; Verify the validity of the number of partition keys (when (or (> key-count max-key-count) (and (> key-count 1) (string= "RANGE COLUMNS" (partition-method first-part)))) @@ -124,8 +132,10 @@ "~a.~a's partition key num(~d) exceed max value(~d), create as normal table" (schema-name (partition-schema first-part)) (table-name table) key-count max-key-count) - (setf is-skip 1) - ) + (setf is-skip 1)) + + ;; When partition_method is "KEY" or "LINER KEY", the type of the KEY is validated. + ;; The whitelist of the type is white-list, as defined above (when (and (not is-skip) (or (string= (partition-method first-part) "KEY") @@ -148,8 +158,9 @@ :always is-skip :do (progn (when (string= (column-type-name column) white-type) - (setf is-skip NIL))))))))) - ) + (setf is-skip NIL)))))))))) + + ;; Initializes the corresponding statement based on the obtained partition information (defparameter statement "") (unless is-skip (setf statement (concatenate 'string statement " PARTITION BY ")) @@ -167,12 +178,11 @@ (setf is-skip 1) (log-message :warning "Unknown partition type") (log-message :warning "Unknown partition type: ~s, create this table(~s.~s) as non-part table" - (partition-method first-part) (partition-schema partition) (table-name table)))) - )) + (partition-method first-part) (partition-schema partition) (table-name table)))))) + (unless is-skip (setf statement - (concatenate 'string statement (format nil "(~a) (" (remove #\` (partition-expression first-part))))) - ) + (concatenate 'string statement (format nil "(~a) (" (remove #\` (partition-expression first-part)))))) (loop :for partition @@ -206,10 +216,10 @@ (log-message :warning "Unknown partition type") (log-message :warning "Unknown partition type: ~s, create this table(~s.~s) as non-part table" (partition-method first-part) (partition-schema partition) (table-name table))))))) - (unless is-skip - (format s "~a)" statement) - ) - ) + + ;; Append the statement after the constructor sentence + (unless is-skip (format s "~a)" statement))) + (when (table-tablespace table) (format s "~%TABLESPACE ~a" (table-tablespace table))) -- Gitee