末日逃亡(僵尸射击
240.4MB · 2026-04-01
浣滆€?* 涓夌嚎绋嬪簭鍛榎nTags * MySQL Doris PG 杈炬ⅵ 閲戜粨鍏抽敭璇?* SeaTunnel銆丏olphinScheduler銆佷俊鍒涖€佸浗浜с€佽揪姊︺€佷汉澶ч噾浠?/p>
[toc]
棣栧厛闇€瑕佺‘淇漇eaTunnel鑳芥甯歌繍琛岋紝闇€瑕佸湪Linux鏈嶅姟鍣ㄥ簱鎴朩indows鍛戒护琛屼笂杩涜楠岃瘉锛屽熀纭€鐨凷eaTunnel鏈湴鐨刏eta寮曟搸鍙互姝ｅ父宸ヤ綔杩愯銆?/p>
濡傜敤 Windows锛屽彲鑳戒細鍑虹幇SeaTunnel浠婂ぉ鑳借繍琛岋紝鏄庡ぉ涓嶈兘杩愯鐨勭壒娈婃儏鍐碉紙鎶ラ敊鍐呭鏄€滄壘涓嶅埌鎴栨棤娉曞姞杞戒富绫烩€濓級锛涙垜娌℃湁褰诲簳瑙ｅ喅锛屼絾鍦ㄧ綉涓婃壘鐨勬柟妗堝ぇ閮ㄥ垎閮芥槸java鐜鍙橀噺璁剧疆鐨勬儏鍐碉紝杩樻湁灏辨槸鍏虫帀鍛戒护绐楀彛閲嶆柊鎵撳紑銆備絾鍋跺皵鏈夋満浼氱‘瀹炲啀娆″嚭鐜帮紝闅斿ぉ灏辨病浜嬩簡銆傜濂囩殑绯荤粺锛?/p>
-- 瀹樻柟妯℃澘涓€娆￠€歕nenv { parallelism = 1 job.mode = "BATCH"}source { Jdbc { url = "jdbc:dm://e2e_dmdb:5236" driver = "dm.jdbc.driver.DmDriver" connection_check_timeout_sec = 1000 user = "SYSDBA" password = "SYSDBA" query = """select * from "SYSDBA".e2e_table_source""" }}sink { Jdbc { url = "jdbc:dm://e2e_dmdb:5236" driver = "dm.jdbc.driver.DmDriver" connection_check_timeout_sec = 1000 user = "SYSDBA" password = "SYSDBA" query = """INSERT INTO SYSDBA.e2e_table_sink (DM_BIT, DM_INT, DM_INTEGER, DM_PLS_INTEGER, DM_TINYINT, DM_BYTE, DM_SMALLINT, DM_BIGINT, DM_NUMERIC, DM_NUMBER, DM_DECIMAL, DM_DEC, DM_REAL, DM_FLOAT, DM_DOUBLE_PRECISION, DM_DOUBLE, DM_CHAR, DM_CHARACTER, DM_VARCHAR, DM_VARCHAR2, DM_TEXT, DM_LONG, DM_LONGVARCHAR, DM_CLOB, DM_TIMESTAMP, DM_DATETIME, DM_DATE, DM_BLOB, DM_BINARY, DM_VARBINARY, DM_LONGVARBINARY, DM_IMAGE, DM_BFILE)VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""" }}鎵嬪啓query姝ｅ父锛屽姩鎬佸嵈涓嶈锛?code>generate_sink_sql =true涓嶈銆傛渶缁堥渶瑕佽拷鍔犱笂鏁版嵁搴撳悕绉帮紝鑰屼笖婧愯〃鏄疍oris琛ㄩ兘鏄皬鍐欙紝鑰岀洰鏍囪〃鏄揪姊﹁〃搴撹〃瀛楁閮芥槸澶у啓锛屾墍浠ヤ細鎶ヨ〃涓嶅瓨鍦ㄣ€?/p>
闇€瑕佽拷鍔犺浆鎹㈠姛鑳斤紝瀛楁澶у皬鍐欎篃闇€瑕佽浆鎹€?/p>
琛ㄨ浆鎹㈤渶瑕佷娇鐢═ransform锛屽苟涓旇拷鍔犳簮琛ㄥ埆鍚嶄笌鐩爣琛ㄨ〃鍒悕锛屾柟渚挎搷浣溿€?/p>
transform { TableRename { plugin_input = "source_doris" plugin_output = "desc_dameng" convert_case = "UPPER" }}瀛楁杞崲
sink { Jdbc { plugin_input = "desc_dameng" url = "jdbc:dm://dmhost:2070?schema=X_Y_Z_KFC" driver = "dm.jdbc.driver.DmDriver" user = "X_Y_Z_USE" password = "123456" database = "DAMENGKFC" table = "X_Y_Z_KFC.${table_name}" generate_sink_sql = true field_ide="UPPERCASE" schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST" data_save_mode="DROP_DATA" dialect ="Dameng" --濂藉儚娌″暐瀹為檯鎰忎箟锛堣鏄細鏍规嵁jdbc杩炴帴涓叉帹绠楋級}}浣庣増鏈笉鏀寔杈炬ⅵ寤鸿〃(2.3.8灏辨病鏈夎揪姊︾殑ddl鍒涘缓琛ㄧ殑鏂规硶锛岃揪姊︽暟鎹簱sink鍐欏叆鏃禿dl鑷姩寤鸿〃鏂规硶鏄湪2.10鍚庢墠杩藉姞鐨?锛屾渶缁堝崌绾у埌2.3.10骞朵笅杞藉搴旂殑connector杩炴帴鍖呰繕鏄笉琛屻€?/p>
鐗堟湰鍗囩骇鏈€鏂颁笌鏇存柊鏈€鏂扮殑杩炴帴鍖咃紙涓棿涓嬭浇鏃堕棿澶暱锛屼娇鐢ㄤ簡浠巑aven涓婄洿鎺ユ墜鍔ㄤ笅杞界殑鍖咃紝鏈€鍚庡浜嗕竴涓嬪ぇ灏忛兘涓€鏍枫€傛病鏈夐棶棰樸€傘€傘€傦級鍙堝崌绾у埌浜?.3.12鐗堟湰涔熸槸鏃犳硶鑷姩寤鸿〃銆?/p>
Dialect鏂硅█鏄剧ず鎸囧畾涔熶笉琛岋紙鍖呮嫭鐩稿簲鐨刲ib鍖呬篃閮藉姞涓婁簡銆傘€傘€傝繕鏄笉琛岋紝鐢氳嚦鎬€鐤戣繃杈炬ⅵ鐨勯┍鍔ㄥ寘鏈夐棶棰橈紝鏄惁闇€瑕佹壘鍟嗗巶瑕佷釜椹卞姩鎵嶈兘鐢級銆?/p>
鏈€缁堥獙璇佷笌瀛楁娉ㄩ噴鏈夊叧锛岃〃娉ㄩ噴鐩存帴灏辨墧浜嗘牴鏈笉寤恒€傚彧瑕佽〃涓湁瀛楁灏辨姤commont 璇硶瑙ｆ瀽闂銆?/p>
鍦ㄦ祴璇曠幆澧冩病鏈夐棶棰橈紝鍦ㄧ敓浜х幆澧僁oris鐗堟湰鍗囩骇浜嗙珶鐒舵姤璇诲彇閿欒浜嗐€傘€傘€?娴嬭瘯鐜鏄疍oris 2.1.3锛岀敓浜ф槸2.1.9鐗堟湰锛屼及璁℃槸鍝湁宸紓銆?
鍙傝€僊ySQL瀵煎叆杈炬ⅵ锛屼娇鐢↙inux鐨勮剼鏈鐞哾ump鐨凷QL鑴氭湰 銆?/p>
鎶婂瓧娈靛拰琛ㄦ敞閲婂幓闄わ紝浣跨敤AI澶勭悊浜嗕竴涓婂崍锛屽彧鑳芥妸琛ㄥ瓧娈垫敞閲婂幓鎺夈€備絾鏄棤娉曟妸瀛楁娉ㄩ噴鍜岃〃娉ㄩ噴鍗曠嫭寮勫埌涓€涓剼鏈腑锛屽彧鑳界敓鎴愪竴涓墍璋撶殑绾噣寤鸿〃璇彞銆?/p>
浣嗙獊鐒跺彂鐜板鍑虹殑鏁版嵁绫诲瀷鑲畾鍦ㄨ揪姊︿腑鏃犳硶鎵ц锛岄渶瑕佽浆鎹€傚搴斿埌鍚勭涓嶅悓鐨勭被鍨嬶紝杩欎釜瀵瑰簲鍐嶇敤鎵嬪伐鍋氫竴閬嶏紝鑰冭檻鏀惧純姝ゆ柟妗堛€?/p>
閫氳繃schema_info鐩存帴鐢╯ql鎷煎嚭涓€鍫嗘竻娉ㄩ噴鐨勮鍙ャ€?/p>
杩樻湁涓皬娉㈡姌锛屽樊鐐硅繖涓柟妗堜篃涓嶈兘鐢ㄤ簡銆傚氨鏄疍oris鐨勯粯璁ゅ€硷紝寮€濮嬭浆鎹㈡椂浣跨敤鐨凷QL
ALTER TABLE dwd_X_Y_base MODIFY COLUMN filedA1 varchar(255) COMMENT "";鎸囧畾浜嗗瓧娈电被鍨嬶紝褰撴湁瀛楁last_update鏈夐粯璁ゅ€兼椂涓嶅厑璁镐慨鏀广€傚悗鏉ヤ粩缁嗘煡浜嗘煡瀹樼綉鏂囨。锛孌oris杩樻槸鍋氫簡浜虹殑锛屾湁涓撻棬鍗曠嫭鍘绘敞閲婄殑璇彞锛屼笉鍔犲瓧娈电被鍨嬪氨琛屼簡銆?/p>
鍒╃敤鐜版湁鐨勫浠藉簱锛屾垨鑰呯洿鎺ラ噸鏂板仛涓复鏃跺浠斤紱鎬濊矾灏辨槸閫氳繃澶囦唤搴撳幓鎶婅〃寤轰笂锛屽啀鍒囧埌姝ｅ紡搴撳幓鍋氱湡瀹炵殑鏁版嵁鍚屾銆?/p>
a. 閫夋嫨琛ㄨ寖鍥淬€傦紙鍙彇Xyz鐨勫簳灞傛暟鎹〃锛岀敤浜庡垎涓氬姟鍘诲仛鍚屾SeaTunnel鎷煎摢浜涜〃鍋氬悓姝ワ級
SELECT * FROM information_schema.TABLES WHERE TABLE_SCHEMA = 'data_test_backup' AND ( TABLE_NAME like 'ods_xyz_%' or TABLE_NAME like 'dwd_xyz%') ;b. 閫夋嫨鍘婚櫎瀛楁娉ㄩ噴鐨勫唴瀹广€?/p>
SELECT CONCAT('ALTER TABLE ', TABLE_SCHEMA,'.',TABLE_NAME, ' MODIFY COLUMN ', COLUMN_NAME, ' ', ' COMMENT "";') AS alter_sqlFROM information_schema.columns WHERE TABLE_SCHEMA = 'data_test_backup' AND (table_name LIKE 'ods_xyz_%' OR TABLE_NAME LIKE 'dwd_xyz%') AND LENGTH(COLUMN_COMMENT) > 0;c. 澶嶅埗鍑烘竻闄ゅ瓧娈佃剼鏈紝鍦ㄥ浠藉簱涓婄洿鎺ユ墽琛屻€?/p>
alter table ods_xyz_1001_base modify column filed1 comment "";alter table ods_xyz_1001_base modify column filed2 comment "";.....ctrl+A 銆?ctrl +R 鍏ㄩ€夎繍琛屻€?/p>
鏂板缓xyz_low1.conf璇诲彇澶囦唤搴撳缓琛ㄥ垵濮嬪寲鍒拌揪姊︼紝鏈妸澶氳〃鏀规垚姝ｅ垯鍘诲尮閰嶏紝鏄柟渚胯皟璇曟壘閿欍€傜洿鎺ユ妸琛ㄥ悕鎵旂粰AI鐢熸垚table_path鏁扮粍锛屼篃鏂逛究浠ュ悗鍋氱湡澧為噺鏃讹紝鐩存帴鍦╯ql涓拷鍔犻檺鍒舵潯浠躲€?/p>
env { parallelism = 2 job.mode = "BATCH"}source { Jdbc { plugin_output = "source_doris" url = "jdbc:mysql://backuphost:9030/data_test_backup" driver = "com.mysql.cj.jdbc.Driver" connection_check_timeout_sec = 100 user = "XXX" password = "******" table_list = [ { table_path = "data_test_backup.dwd_xyz_1001_base" query = "select * from data_test_backup.dwd_xyz_1001_base" }, { table_path = "data_test_backup.dwd_xyz_1002_base" query = "select * from data_test_backup.dwd_xyz_1002_base" }, .....]}}transform { TableRename { plugin_input = "source_doris" plugin_output = "desc_dameng" convert_case = "UPPER" }}sink { Jdbc { plugin_input = "desc_dameng" url = "jdbc:dm://101.2.3.4:2026?schema=X_Y_Z_KFC" driver = "dm.jdbc.driver.DmDriver" user = "X_Y_Z_USE" password = "123456" database = "DAMENGKFC" table = "X_Y_Z_KFC.${table_name}" generate_sink_sql = true field_ide="UPPERCASE" schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST" data_save_mode="DROP_DATA" dialect ="Dameng"}}澶嶅埗澶囦唤搴撶殑閰嶇疆鏂囦欢锛屼慨鏀规暟鎹簱ip鍦板潃绔彛瀵嗙爜绛変俊鎭紝灏卞彲鐩存帴杩愯浜嗐€?/p>
杩藉姞琛ㄦ敞閲娿€?/p>
SELECT CONCAT('COMMENT ON TABLE ', upper(TABLE_NAME), ' IS ''', TABLE_COMMENT, ''';') FROM information_schema.TABLES WHERE TABLE_SCHEMA = 'data_test' AND (table_name LIKE 'ods_xyz_%' OR TABLE_NAME LIKE 'dwd_xyz_%')鍦ㄨ揪姊︽暟鎹簱涓婃墽琛岋紒鎶婅嚜鍔ㄥ缓琛ㄧ殑琛ㄦ敞閲婅ˉ鍑烘潵銆?/p>
鏈夋潯浠惰拷鍔犲瓧娈垫敞閲?褰撴椂浠诲姟鎬ワ紝鏈潵鍙湡浣犳噦鐨?銆?/p>
涓埆璇彞鏈夐棶棰樼殑闇€瑕佽皟鏁翠慨鏀逛竴涓嬶紝璁板緱閲嶆柊鎶婂厛琛ㄥ垹闄や簡锛屽啀鏀归厤缃枃浠朵腑鐨勫唴瀹广€?/p>
{ table_path = "data_test_backup.ads_xyz_1001_agg" query = "select label_type,`sum(total)` as 'totalnum',sord_num from data_test_backup.ads_xyz_1001_agg" },Windows銆丩inux鐩存帴鑴氭湰瀹氭椂锛屾垨鑰呴泦鎴恉s杩涜閰嶇疆鍙鍖栦换鍔°€?/p>
Windows涓嬬殑bat鑴氭湰
@echo offrem 鍏堝垏鍒?SeaTunnel 鐨?bin 鐩綍cd /d "E:\apache-seatunnel-2.3.12-bin\apache-seatunnel-2.3.12\bin"rem 鎵ц浣滀笟seatunnel.cmd --config ./job/xyz_low.conf -m local閫氳繃瀛︿範杈炬ⅵ鏁版嵁搴擄紝绗旇€呭彂鐜板畠鏈韩灏辨槸Oracle鐨勯瓟鏀圭増鏈紝鏈夌偣鍍忔妸PG鍜孫racle鎹忓湪浜嗕竴璧凤紝鍔犱簡涓狿G鐨凷chema锛岃娉曞叏鏄疧racle鐨勩€傜瑪鑰呬富瑕佸埄鐢ㄤ簡SeaTunnel鐨勮嚜鍔ㄥ缓琛ㄥ姛鑳斤紝鐗瑰埆鏄瓧娈电被鍨嬫槧灏勮浆鎹㈣妭鐪佷簡澶ч噺鏃堕棿锛屼絾鐮旂┒鐨勬椂闂翠篃涓嶇煭銆?/p>
鍚屾椂锛岀瑪鑰呬篃鍙戠幇浜嗕竴浜涢棶棰橈紝姣斿琛ㄦ敞閲婁細涓㈠純锛岃繖浜涜繕濂斤紝鍙嶆灏辨槸涓€娆℃€х殑浜嬫墜鍔ㄨˉ涓€涓嬶紝鐩存帴鐢╯ql鐢熸垚涓€涓嬭剼鏈嵆鍙€?/p>
鍦ㄦ姤閿欒皟璇曟柟闈紝浼间箮鐢变簬绾跨▼鐨勯棶棰橈紝浼氭妸鍚岀嚎绋嬬殑鍏跺畠琛ㄧ殑鏃犻敊鐨勫唴瀹规姤鎴愬紓甯告墦鍗板嚭鏉ャ€?/p>
杩樻湁灏辨槸鍏充簬琛ㄧ粨鏋勭殑闂锛岃娉ㄦ剰璋冭瘯杩囩▼涓墜鍔ㄥ垹闄よ〃锛屽洜涓洪粯璁や娇鐢ㄧ殑鍙傛暟鏄?code>schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST"锛屽鏋滆〃宸茬粡瀛樺湪涓嶄細鍐嶅垱寤鸿〃锛屽鏄撻€犳垚寤虹殑琛ㄦ湁闂锛岃€屽彂鐢熷鎬殑寮傚父鎶ラ敊銆?/p>
鍚庣画骞冲彴鍙堥渶瑕佷笌绗笁鏂圭墿鑱旂郴缁熷仛鏁版嵁瀵规帴锛屽氨鐩存帴鍒╃敤浜咲oris stream load鎶€鏈潵瀹炵幇锛屽垎浜竴涓嬬粡楠岋細鏈€缁堥渶瑕佸皢http鎺ュ彛澶栫綉鏆撮湶鐨勫湴鍧€鏄疍oris鐨刡e绔彛锛岃€岄潪fe绔彛銆?/p>
涓棿杩橀獙璇佷簡涓€涓媺鐨勬柟寮忥紝灏辨槸鍒╃敤SeaTunnel鐨刪ttp杩炴帴鍣紝鍘绘媺鏁版嵁銆傝繖閲屾湁涓皬闂锛屽氨鏄渶瑕佸仛閴存潈锛屾湁鏃堕棿浼氬啀鍋氫釜鍒嗕韩銆傦紙鏂规硶寰坙ow浣嗛獙璇佸彲琛屻€傦級
淇″垱灏辨槸鎴戜汉鐢熺殑鑷虫殫鏃跺埢锛屽垰缁忓巻浜嗚揪姊﹀張寰楀紕Kingbase锛屼絾鏈€缁堝鑷繁涓汉鎴愰暱杩樻槸鏈夊姪鍔涚殑锛屼笉璇翠俊鍒涙暟鎹簱鎬庝箞鍏煎鐨勫悇绉嶉棶棰樺惂锛屽湪鏃朵笅杩欎釜鐜鎹釜瑙掑害鐪嬶紝杩欏彲鑳藉氨鏄竴绉嶁€滄妧鏈鍨掆€濄€備篃娌℃椂闂村啓鍐呴儴鎶€鏈枃妗ｄ簡锛岀洿鎺ヤ粠澶村洖蹇嗗惂銆?/p>
鍏堣寮€鏀炬€ingbase鑷冲皯姣旇揪姊﹀己锛屽畼缃戠粰涓嬭浇瀹夎绋嬪簭鍖咃紝鍖呮嫭瀹夎鐗堝拰docker鐗堟湰锛岃繕鍙互鍏嶈垂鐢宠娴嬭瘯鐨勬巿鏉冭瘉涔︼紝寮€鍙戞巿鏉冩渶澶氭湁1骞寸殑璇曠敤鏈熴€傝繖涓€鐐瑰氨鏁炰寒銆佸眬姘斻€傞鍏堝拰韬竟鐨勫墠鍚屼簨锛堢幇鍦ㄨ繕鏄ソ鏈嬪弸锛夋墦鍚簡涓€涓嬶紝浠栦滑涔嬪墠璇曠敤杩囷紝澶ф灏辨槸Kingbase鏄釜濂楀３锛屽簳灞傛槸pg锛屾敼浜嗘敼鍑犱釜鍑芥暟锛岃寮€鏀炬€ф湁涓彨鐎氶珮鐨勬洿寮€鏀撅紝鍩烘湰娌℃湁榄旀敼鐨勶紝鏈潃鍘熺敓鐨勪竴鑷磋繘琛屼簡浜屽紑銆?/p>
鐒跺悗杩愮淮澶у摜涓変笅浜旈櫎浜屽氨鎶奷ocker鎷夎捣鏉ワ紝楂橀珮鍏村叴閫夋嫨浜嗕笅MySQL妯″紡锛岀粨鏋淢ySQL鐨勯┍鍔ㄤ笉鑳界洿鎺ヨ繛锛屽繀椤昏鐢↘ingbase鐨勫師鐢燂紝涓棿鐪佺暐鍚勭闂锛屾渶缁堝張瑁呬簡涓€涓猵g妯″紡鐨勩€?/p>
姒傚康灏变竴涓?/strong>锛欿ingbase鏈夊绉嶅吋瀹规ā寮忥紝mysql/pg/sqlserver浠€涔堢殑銆傘€傘€傜悊璁轰笂涓嶈€冭檻杩欎釜鍏煎妯″紡鐢↘ingbase鍘熺敓鐨勯┍鍔ㄨ偗瀹氶兘鑳借繛鎺ャ€傚鏋滅煡閬撳叿浣撶殑鍏煎妯″紡锛屽彲浠ュ皾璇曠敤鍏煎鐨勯┍鍔ㄨ繛鎺ャ€傚pg妯″紡鐩存帴鐢╬g椹卞姩灏卞彲浠ヨ繛鎺ワ紝浣哅ySQL妯″紡Navicat灏变笉鑳界敤MySQL鐨勯┍鍔ㄨ繛鎺ャ€?/p> KSQL鏄疜ingbase鑷繁鐨勮繛鎺ュ伐鍏凤紝鏈夊繀瑕佷篃瀹夎涓€涓紝瀹冪殑椹卞姩灏辨槸鐢ㄧ殑Kingbase鍘熺敓鐨勯┍鍔ㄣ€?/p> 鍚姖MySQL鍏煎妯″紡涓嶅ソ鐢紝鍜卞氨鐢ㄥ簳灞傚師鐢熺殑鏈€绋冲畾浜嗭紝褰撳勾kmx鐩存帴鐢ㄧ殑Cassandra璇荤殑濡ュΕ鐨勫ソ銆傞偅灏卞噯澶囩敤Kingbase鐨刾g鍏煎妯″紡鍋氫负婧愬拰鐩爣浜?/strong>銆?/p> 鍦⊿eanTunnel瀹樻柟鏂囨。涓婃煡浜嗕竴涓嬶紝鏀寔Kingbase锛屼絾鏄紝浣嗘槸锛屼絾鏄紝鍙湁閮ㄥ垎绫诲瀷鍏煎锛佸張鍦ㄦ妧鏈兢閲屽湀浜嗕竴涓?strong>panda jdbc:kingbase8鐨勪笉褰掕矾灏卞紑濮嬩簡....锛堣繖閲宻ource鍜宻ink鐨勫簱閮芥槸Kingbase鐨刾g妯″紡锛?/p> 涓€鍒囬『鍒╋紝涓嶅埌鈥?0鈥濆垎閽熷氨鎼炲畾浜嗭紱褰撴椂娴嬭瘯鐨勫皬閬楁喚鏄笉鑳絪chema_save_mode鑷姩寤鸿〃銆傚湪浜ゆ祦缇ら噷鍚愭Ы浜嗕竴涓嬶紝涔熸劅璋㈣繀鍝ュ効鍜岃タ闂ㄥ垎浜粡楠屽拰鎯虫硶锛侊紒 鍚庢潵panda澶т浆瑕佺粰Kingbase绔媐lag璇村彲浠ユ敮鎸侊紝鎴戞槸娴嬭瘯浜嗕笉琛岋紱panda浣Kingbase鏄户鎵縫g鐨勪唬鐮侀兘鏀寔锛岃繕鎻愰啋鍢卞拹source涓嶈兘鐢╭uery锛屾棤娉曡嚜鍔ㄥ缓琛紝瑕佺敤 閲嶇偣鏄細鈥滅敤pg杩炴帴鍣ㄦ槸鍙互鍦帮紝濡傛灉浣燢ingbase鏈韩鏄痯g鍏煎妯″紡 閭ｅ彲浠ョ敤pg鐨勶紝鍙鍏冩暟鎹鏌ヨ兘閫氳繃銆傞偅灏辨崲鎴恜g椹卞姩鍜岄厤缃瘯璇曗€濓紝缁撹灏辨槸鈥滄妸kingbase鐨刾g妯″紡灏卞綋鎴恓dbc鐨刾g鐢ㄢ€?/strong>锛岃€屼笖鍙互鑷姩寤鸿〃绛夊弬鏁伴兘鑳界敤浜嗐€?strong>鈥減g鏀寔鍟ュ畠鏀寔鍟モ€?/strong>銆?/p> 婊″績娆㈠枩鐨勪笅鐝紝涓€鍒囬兘澶『鍒╀簡銆傘€傘€?/p> 涓氬姟涔熻鍋氫俊鍒涘噯澶囷紝閭ｅ府瀛愯€佸彜钁ｅ氨鍜浜嗘垜杩欑浼犱唬鐮佸氨鏄疢ySQL锛屼俊鍒涙垜涔熸槸鐢ㄩ噾浠撶殑MySQL鍏煎妯″紡锛侊紒锛侊紒锛佹垜杩樻彁鍓嶅垎浜簡楠岃瘉缁撹锛屽憡璇変粬浠帹鑽愮敤pg妯″紡锛屽彲鏄汉瀹朵笟鍔″氨鏄繖涔堟í锛岃鎴戜滑鎹g锛屾垜杩欏畬鍏ㄤ笉鎺ュ彈锛屾垜鎵鹃瀵煎幓锛侊紒锛侊紒锛佸彲鎯宠€岀煡鐨勭粨鏋滐紝寮勪笉浜嗗氨鏄綘浠妧鏈笉琛屻€傛垜杩欒鍘嬩竴涓嬪瓙灏变笂鍘讳簡锛孈#锟?鈥︹€?amp;%鈥︹€?amp;鈥︹€?amp;锛?amp;锟?&鈥︹€︼紙**锛堬級锟?&鈥︹€?/em>锛堬級锟?锟? 杩橩ingbase鐨勫吋瀹筂ySQL妯″紡鑲畾鏄被鍨嬫湁闂鍟婏紝杩欏彲鎬庝箞鍔烇紵璧剁揣鎵惧叾瀹冨姙娉曞惂銆傜綉涓婃壘浜嗘湁浠€涔圖atamover锛孌ataX( 鑰佸浼?锛岃繕鏈変竴鐩村叧娉ㄦ病鐢ㄨ繃鐨凾is璧剁揣寮勮繃鏉ヨ瘯鍚э紝鏃堕棿绱т换鍔￠噸锛孴is鏈塪ocker鐗堟湰锛岃刀绱ф媺璧锋潵銆傝瘯浜嗕竴涓嬭繕鐪熻锛岀偣鐐圭偣灏卞紕濂戒竴寮犺〃锛侊紒锛佽〃涔熷缓涓婁簡鏁版嵁涔熷杩囧幓浜嗭紝鎸哄ソ銆?/p> 椤哄埄鍚?...娌¤繃涓€浼氾紝璇磋〃鐨勫瓧娈甸兘鏄ぇ鍐欑殑锛孠ingbase榛樿鏄尯鍒嗗ぇ灏忓啓鐨勫拰pg涓€鏍枫€備絾鏄彲浠ラ€氳繃鏁版嵁搴撳垵濮嬪寲鏃舵寚瀹氾紝Docker涓嬮潰鎸囧畾閭ｄ釜鍙傛暟鏄捣涓嶆潵鐨勶紝杩愮淮澶у摜璇村彧鑳藉～pg锛屽～涓嶄簡鍏跺畠鐨勩€傚張鏄釜涓ゅご鍫垫鐨勬儏鍐碉紝鍍忎笉鍍忚揪姊︼紵銆傘€傘€?/p> 绠€鍗曠湅浜嗙湅Tis搴曞眰鐢ㄧ殑DataX锛屽缓琛ㄨ鍙ュ彲浠ヨ嚜宸变慨鏀瑰瓧娈靛悕鍙樺皬鍐欙紝浣嗘槸DataX鐨勮剼鏈笉璁╂敼锛岀洿鎺ユ嫹鍑烘潵鍦―ataX涓婃墽琛屾湁闂锛岀湅涓嶆噦鐨勯敊璇€傛病鏃堕棿浜嗙爺绌朵簡銆傘€傘€?/p> 鏅曟檿蹇藉拷涓€涓嬪崍锛屽帇鍔涘ぇ鍚冪⒊姘村锛屾劅瑙夊埌鍘嬪姏涓庣敓娲荤殑褰卞搷浜嗭紝灏辫鑷繁璋冭妭銆傚伐浣滃彧鏄伐浣滐紝杩樻湁鐢熸椿銆傞噸鏂拌皟鏁撮ギ椋燂紝鏃╀笂鏈夋椂闂磋繕鎶婂閲岀殑姣涘肪娲椾簡娲楋紝蹇冩儏鎷夋弧鍘讳笂鐝€?/p> 鏉ュ惂锛屽啀璇曚竴鎶婅€佹湅鍙婼eaTunnel锛佽繕鏄€佷笁鏍凤紝connector閲嶄笅锛岄┍鍔ㄩ噸鏀撅紝鎵ц鏂囦欢缂栫爜闂銆備竴鍏充竴鍏宠繃鍛楋紝MySQL鍏煎绫诲瀷鏈夐棶棰橈紝鎴戝厛璺宠繃閭ｄ釜瀛楁鐩存帴鍐欐鍑犱釜鍒楋紝鍏堣窇涓€鎶婄粰缁欒嚜宸变俊蹇冦€?/p> **娉ㄦ剰鐜鏈夋敼鍙橈細**192.168.0.31:4321 鐨凨ingbase鏄疢ySQL鍏煎妯″紡锛?92.168.0.119:4322鏄疜ingbase鐨刾g鍏煎妯″紡銆?/p> 鎵€浠ource瑕佺敤Kingbase鐨勫師鐢熷幓璇伙紝瀛楁杞皬鍐欑殑闂锛岄€氳繃SQL鍏堝皾璇曡В鍐筹紝澶у姏鍑哄杩癸紝杩欎簺涓墰椹殑浜嬫墧缁橝I寮勶紱sink淇濈暀鍘熸潵鐨刾g涔熸病浜嬨€?/p> 缁忚繃9*9=81娆¤皟璇曪紝涓€鎶婅繃浜嗐€傞珮楂樺叴鍏存壘杩愮淮澶у摜璇存垚浜嗘垚浜嗭紝杩愮淮澶у摜闂簡涓€鍙モ€渋nt鍨嬬殑鎬庝箞瑙ｅ喅鐨勨€濓紵鎴?$%&&,鎴戠粫杩囧幓浜嗐€傘€傘€傘€傛檿浜嗗繕浜嗕釜骞插噣銆?/p> 淇″績鏈変簡锛屽彲鐜板疄灏辨槸杩欎箞鍐板喎锛宨nt绫诲瀷杞崲澶辫触....AI璇存寚瀹歴ource鐨勮〃缁撴瀯绫诲瀷锛屼笉绠＄敤....sql杞崲绫诲瀷涔熸病璇曟垚鍔?....涓嶈锛屾湇杞紝鑺辩偣閽变拱Kingbase鐨勪骇鍝佸惂銆傛渶澶氫篃灏辫繖鏍蜂簡銆?/p> Panda浣殑閭ｅ彞"鐢╬g杩炴帴鍣ㄦ槸鍙互鍦?quot;锛屾垜鍙堝啀娆′粩缁嗙悊瑙ｄ簡涓€涓嬨€傛槸涓嶆槸鏈変粈涔堟病鐞嗚В鍒帮紵鎴戠敤pg椹卞姩璇讳釜Kingbase鐨凪ySQL鍏煎妯″紡锛屽啀璧屼竴鎶婏紵 鏈€鍚庣粨灞€浜嗚偗瀹氭槸杩囦簡锛屽啀tm涓嶈繃杩欐枃绔犲氨娌″繀瑕佸啓杩欏潡浜嗐€傚悗鏉ユ嬁Navicat鐩存帴杩炰簡涓€涓婯ingbase鐨凪ySQL鍏煎妯″紡锛屼篃鑳借繛涓娿€?锟?锛屽師鏉ユ槸鑷繁缁曡繙浜嗐€?/p> 璧剁揣鍒嗕韩缁欑兢閲岀殑灏忎紮浼达紝鍙堝拰panda浣皥浜嗕綋浼氾紝鈥滈偅鎸哄ソ鍟娾€濓紝鍘熸潵涓栫晫鐪熺殑寰堝ぇ锛屾垜浠彧鍦ㄨ嚜宸辩殑浜曢噷銆傛湁浜涗簨鍙槸鑷繁娌¤杩囷紝浣嗗苟涓嶄唬琛ㄨ繖涓笘鐣屼笂娌℃湁銆?/p> 鈥? 2026.1.21 涓夌嚎绋嬪簭鍛?/p>", 2. 棰勬湡璁炬兂
env { parallelism = 2 job.mode = "BATCH"}source { Jdbc { driver = "com.kingbase8.Driver" url = "jdbc:kingbase8://192.168.0.31:4322/sourcedb" user = "kingbase" password = "123456" query = "select * from source_user_detail" }}sink { jdbc { url = "jdbc:kingbase8://192.168.0.119:4322/targerdb" driver = "com.kingbase8.Driver" user = "kingbase" password = "123456" generate_sink_sql = true database = targerdb table = public.target_user_detail #schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST" data_save_mode="APPEND_DATA" }}tabl_path鏄釜鍧戯紝璁╂垜璁板埌鏂囩珷閲屾彁閱掑ぇ瀹?/strong>锛屸€滈€犵鏇村浣跨敤鑰呪€濄€傛渶缁坧anda浣彲鑳芥煡浜嗘煡婧愮爜纭浜嗘墦鑴革紝鈥淜ingbase鍦ㄥ缓琛ㄩ偅鍧楁病閫傞厤鈥濓紝浣嗚繖涓嶆槸閲嶇偣銆?/p>source { Jdbc { driver = "org.postgresql.Driver" url = "jdbc:postgresql://192.168.0.31:4322/sourcedb" user = "kingbase" password = "123456" table_path = "sourcedb.public.source_user_detail" }}sink { jdbc { url = "jdbc:postgresql://192.168.0.119:4322/targerdb" driver = "org.postgresql.Driver" user = "kingbase" password = "123456" generate_sink_sql = true database = targerdb table = public.target_user_detail schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST" data_save_mode="APPEND_DATA" }}3. 绐佸彉澶ц浆鎶?/h3>
4. 鑳屽彌
5. 璧屼竴鎶?/h3>
env { parallelism = 2 job.mode = "BATCH"}source { Jdbc { driver = "com.kingbase8.Driver" url = "jdbc:kingbase8://192.168.0.31:4321/kingbase" user = "kingbase" password = "123456" query = "SELECT `ID` AS id,`PARENT_ID` AS parent_id,`DICT_LABEL` AS dict_label,`DICT_VALUE` AS dict_value,...REMARK` AS remark FROM public.dict;" }}sink { jdbc {t url = "jdbc:postgresql://192.168.0.119:4322/datatest" driver = "org.postgresql.Driver" user = "kingbase" password = "123456" generate_sink_sql = true database = datatest table = data_test.dim_busi_dict_001 schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST" data_save_mode="APPEND_DATA" }}env { parallelism = 2 job.mode = "BATCH"}source { Jdbc { driver = "org.postgresql.Driver" url = "jdbc:postgresql://192.168.0.62:4321/kingbase" user = "kingbase" password = "123456" query = "SELECT * FROM public.dict;" }}sink { jdbc {t url = "jdbc:postgresql://192.168.0.119:4322/datatest" driver = "org.postgresql.Driver" user = "kingbase" password = "123456" generate_sink_sql = true database = datatest table = data_test.dim_busi_dict_001ttfield_ide="LOWERCASE" schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST" data_save_mode="APPEND_DATA"t}}
