Skip to content

曾炯豪 / qiyi

Go to a project
Commit efcadefc by 曾炯豪
Options

Initial commit

parents
Inline Side-by-side
Showing with 2497 additions and 0 deletions
qida-dbmigrate-master/.gitignore 0 → 100644
*.log
*.log.*
target
target/*
*/target
*/target/*
*/.project
*/.project/*
*/.settings
*/.settings/*
.idea
*/.idea
*/.idea/*
*.iml
*/*.iml
databases/.skeema
snapshot-repo/*
*.mp4
*/dockerize/*.jar
*/dockerize/lib
*/dockerize/lib/*
qida-dbmigrate-master/README.md 0 → 100644
本代码库假设了迁移所设计的source mysql-server和destination mysql-server均使用https://gitlab.qida.com/qida-b2b-refactoring-serviceapps/hybrid/tree/case/databases/skeemaBak最新commit所表示的schema.
qida-dbmigrate-master/dbmigrate/pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.qida</groupId>
<artifactId>dbmigrate</artifactId>
<version>1.0-SNAPSHOT</version>
<name>dbmigrate</name>
<!-- FIXME change it to the project's website -->
<url>http://www.example.com</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.7</maven.compiler.source>
<maven.compiler.target>1.7</maven.compiler.target>
<slf4jVer>1.7.25</slf4jVer>
<logbackVer>1.2.3</logbackVer>
<mysqlConnectorJVer>8.0.20</mysqlConnectorJVer>
<apacheCommonsDbcp2Ver>2.7.0</apacheCommonsDbcp2Ver>
<apacheCommonsPool2Ver>2.8.0</apacheCommonsPool2Ver>
<apacheCommonsLoggingVer>1.2</apacheCommonsLoggingVer>
<apacheCommonsCliVer>1.4</apacheCommonsCliVer>
<fastjson.version>1.2.47</fastjson.version>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<!-- Apache commons cli -->
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>${apacheCommonsCliVer}</version>
</dependency>
<!-- MySQL Connector/J driver -->
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>${mysqlConnectorJVer}</version>
</dependency>
<!-- fastjson -->
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>${fastjson.version}</version>
</dependency>
<!-- DBCP2 connection pool. -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-dbcp2</artifactId>
<version>${apacheCommonsDbcp2Ver}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-pool2</artifactId>
<version>${apacheCommonsPool2Ver}</version>
</dependency>
<!-- Slf4j-api. -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4jVer}</version>
</dependency>
<!-- Logback as Sfl4j-api implementation. -->
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
<version>${logbackVer}</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>${logbackVer}</version>
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongodb-driver</artifactId>
<version>3.4.1</version>
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>bson</artifactId>
<version>3.4.1</version>
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>Mongodb-driver-core</artifactId>
<version>3.4.1</version>
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>4.6.3</version>
</dependency>
</dependencies>
<build>
<finalName>${project.artifactId}</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>./target/lib</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.1.0</version>
<configuration>
<archive>
<manifest>
<!--
Packages the jar with a default main class "com.mycompany.app.App", and
an "implicit classpath recognition source ./lib/".
-->
<addClasspath>true</addClasspath>
<classpathPrefix>lib/</classpathPrefix>
<mainClass>com.qida.dbmigrate.App</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.3</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<createSourcesJar>true</createSourcesJar>
<shadeSourcesContent>true</shadeSourcesContent>
<createDependencyReducedPom>false</createDependencyReducedPom>
<shadedArtifactAttached>false
</shadedArtifactAttached> <!-- Deliberately replacing the original output jar. -->
<shadedClassifierName>shaded</shadedClassifierName>
<artifactSet>
<excludes>
<exclude>junit:junit</exclude>
<exclude>org.springframework:*</exclude>
</excludes>
</artifactSet>
<relocations>
<relocation>
<pattern>com.qida</pattern>
<shadedPattern>com.qida.${project.artifactId}</shadedPattern>
</relocation>
</relocations>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
qida-dbmigrate-master/dbmigrate/src/main/java/com/qida/App.java 0 → 100644
package com.qida;
import org.apache.commons.cli.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.dbcp2.BasicDataSource;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
public class App
{
private static final Logger logger = LoggerFactory.getLogger(App.class);
public static void main(String[] args) {
/*
* In this application, there're 2 "DataSource"s with the same set of schemas, e.g. "foo" and "bar".
* Assume that the structure of "foo" is as follows.
* - (schema) qida_management_platform
* - (table) sys_user // 平台管理员/超级管理员记录
* - (schema) qida_cas_platform
* - (table) cas_user // 所有"用户"（包括平台管理员/超级管理员）的账号密码
* - (table) sys_user_audit
* - ...
* - (schema) clm_base_platform
* - (table) org_company
* - (table) org_user
* - (table) sys_user_role
* - ...
*/
final Options options = new Options();
options.addRequiredOption("sh", "srchost", true, "The source mysql server host(including port), e.g. 192.168.56.101:3306.")
.addRequiredOption("dh", "dsthost", true, "The destination mysql server host(including port), e.g. 192.168.56.102:3307.")
.addRequiredOption("su", "srcuser", true, "The source mysql username.")
.addRequiredOption("du", "dstuser", true, "The destination mysql username.")
.addOption("sp", "srcpassword", true, "The source mysql password.")
.addOption("dp", "dstpassword", true, "The destination mysql password.");
final HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("QidaDataExtractor", options);
Connection fooConnection = null;
PreparedStatement fooStatement = null;
ResultSet fooResultSet = null;
Connection barConnection = null;
PreparedStatement barStatement = null;
ResultSet barResultSet = null;
try {
final DefaultParser parser = new DefaultParser();
CommandLine cl = parser.parse(options, args);
final String sh = cl.getOptionValue("sh");
final String su = cl.getOptionValue("su");
final String sp = cl.getOptionValue("sp");
final String dh = cl.getOptionValue("dh");
final String du = cl.getOptionValue("du");
final String dp = cl.getOptionValue("dp");
logger.info("srchost= " + sh + ", dsthost= " + dh);
final BasicDataSource fooBds = new DataSource("jdbc:mysql://" + sh, su, sp).getBds();
final BasicDataSource barBds = new DataSource("jdbc:mysql://" + dh, du, dp).getBds();
fooConnection = fooBds.getConnection();
fooStatement = fooConnection
.prepareStatement("SELECT * FROM `clm_base_platform`.`org_company` LIMIT 10;");
fooResultSet = fooStatement.executeQuery();
logger.info("id, company_name, short_name, company_code");
while (fooResultSet.next()) {
logger.info(fooResultSet.getInt("id") + ", "
+ fooResultSet.getString("company_name") + " "
+ fooResultSet.getString("short_name") + ", "
+ fooResultSet.getString("company_code"));
}
barConnection = barBds.getConnection();
barStatement = barConnection
.prepareStatement("SELECT 1+1 AS sum;");
barResultSet = barStatement.executeQuery();
while (barResultSet.next()) {
logger.info("bar result:" + barResultSet.getInt("sum"));
}
} catch (SQLException e) {
logger.error("Exception while executing query", e);
} catch (ParseException e) {
logger.error("Exception while parsing args", e);
} finally {
try {
if (null != fooResultSet) {
fooResultSet.close();
}
if (null != fooStatement) {
fooStatement.close();
}
if (null != fooConnection) {
fooConnection.close();
}
if (null != barResultSet) {
barResultSet.close();
}
if (null != barStatement) {
barStatement.close();
}
if (null != barConnection) {
barConnection.close();
}
} catch (SQLException e) {
logger.error("Exception while wrapping up", e);
}
}
}
}
qida-dbmigrate-master/dbmigrate/src/main/java/com/qida/DataSource.java 0 → 100644
package com.qida;
import org.apache.commons.dbcp2.BasicDataSource;
public class DataSource {
// The old driver class "com.mysql.jdbc.Driver" is deprecated in "mysql:mysql-connector-java:8.0.20+", see https://dev.mysql.com/doc/connector-j/8.0/en/connector-j-api-changes.html for more information.
private static final String DRIVER_CLASS_NAME = "com.mysql.cj.jdbc.Driver";
private static final int CONN_POOL_SIZE = 5;
private BasicDataSource bds = new BasicDataSource();
public DataSource(final String dbUrl, final String dbUser, final String dbPassword) {
//Set database driver name
bds.setDriverClassName(DRIVER_CLASS_NAME);
//Set database url
bds.setUrl(dbUrl);
//Set database user
bds.setUsername(dbUser);
//Set database password
bds.setPassword(dbPassword);
//Set the connection pool size
bds.setInitialSize(CONN_POOL_SIZE);
}
public BasicDataSource getBds() {
return bds;
}
}
qida-dbmigrate-master/dbmigrate/src/main/java/com/qida/DemoApp.java 0 → 100644
package com.qida;
import cn.hutool.core.collection.CollUtil;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.mysql.cj.util.StringUtils;
import org.bson.Document;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
public class DemoApp {
public static void main(String[] args) {
List<String> s = new ArrayList<>();
s.add("a");
System.out.println(CollUtil.isNotEmpty(s));
}
}
qida-dbmigrate-master/dbmigrate/src/main/java/com/qida/MongoDBApp.java 0 → 100644
package com.qida;
import com.alibaba.fastjson.JSONObject;
import com.mongodb.MongoClient;
import com.mongodb.MongoCredential;
import com.mongodb.ServerAddress;
import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.MongoIterable;
import com.qida.mongoUtils.MongoDBUtil;
import org.bson.Document;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Scanner;
public class MongoDBApp {
public static void main(String[] args) {
MongoClient src_mongoClient = null;
MongoClient dst_mongoClient = null;
try {
// 键盘输入需要迁移的数据库
System.out.println("请输入mongodb的库名称，重要的事情说三遍！！！");
System.out.println("请输入mongodb的库名称，重要的事情说三遍！！！");
System.out.println("请输入mongodb的库名称，重要的事情说三遍！！！");
// Scanner sc = new Scanner(System.in);
// String databaseName = sc.next();
String databaseName = "tenant";
// 读取json文件
JSONObject jsonObject = getJsonObject();
// ------------------------ 开始操作数据库------------------------------------------------------------
// 数据源src
MongoCredential src_credential = MongoCredential.createCredential("qida", databaseName, "qida".toCharArray());
ServerAddress src_serverAddress = new ServerAddress("192.168.1.242", 3717);
src_mongoClient = new MongoClient(src_serverAddress, Arrays.asList(src_credential));
MongoDatabase src_mongoDatabase = src_mongoClient.getDatabase(databaseName);
MongoIterable<String> src_collectionNames = src_mongoDatabase.listCollectionNames();
// 迁移到的目标数据库dst
MongoCredential dst_credential = MongoCredential.createCredential("qida", databaseName, "qida".toCharArray());
ServerAddress dst_serverAddress = new ServerAddress("192.168.1.242", 3717);
dst_mongoClient = new MongoClient(dst_serverAddress, Arrays.asList(dst_credential));
MongoDatabase dst_mongoDatabase = dst_mongoClient.getDatabase(databaseName);
MongoIterable<String> dst_collectionNames = src_mongoDatabase.listCollectionNames();
// ----------------------------------------------------------------------------------------------------
String message = "";
for (String collection : src_collectionNames) {
message = message + collection + "、";
}
System.out.println(databaseName + "有如下集合： " + message.substring(0, message.length() - 1) + "！！！");
Thread.sleep(3000);
for (String collectionName : src_collectionNames) {
if (!collectionName.equals("Individuation")) {
continue;
}
MongoCollection<Document> collection = src_mongoDatabase.getCollection(collectionName);
/*long count = collectionName.count();
if (count == 0) {
System.out.println(collection + "集合没有数据，准备开始下一个集合的迁移！！！");
continue;
}
System.out.println(collectionName.getNamespace() + "集合共有" + count + "条数据！！！");*/
FindIterable<Document> documents = collection.find();
List<Document> docList = new ArrayList<>();
int i = 0;
for (Document document : documents) {
/*if (i > 20) {
break;
}*/
i++;
docList.add(document);
System.out.println(document);
}
System.out.println("************************************************************************************");
System.out.println("************************************************************************************");
List<String> fieldsToUpdate = isSpecialCol(jsonObject, databaseName, collectionName);
List<Document> documentList = MongoDBUtil.dealDoc(databaseName + "." + collectionName, docList, fieldsToUpdate);
for (Document item : documentList) {
System.out.println(item);
}
}
} catch (Exception e) {
System.err.println("迁移时发生了错误！！！");
System.err.println(e.getClass().getName() + ": " + e.getMessage());
} finally {
src_mongoClient.close();
dst_mongoClient.close();
}
}
// -------------------------工具类方法---------------------------------------------------------------------
private static JSONObject getJsonObject() throws Exception {
String path = MongoDBApp.class.getClassLoader().getResource("mongodb.json").getPath();
String jsonStr = "";
File jsonFile = new File(path);
Reader reader = new InputStreamReader(new FileInputStream(jsonFile), "utf-8");
int ch = 0;
StringBuffer sb = new StringBuffer();
while ((ch = reader.read()) != -1) {
sb.append((char) ch);
}
reader.close();
jsonStr = sb.toString();
JSONObject jsonObject = JSONObject.parseObject(jsonStr);
return jsonObject;
}
// 特殊列的处理方法
private static List<String> isSpecialCol(JSONObject jsonObject, String databaseName, String collection) {
List<String> fieldsToUpdate = new ArrayList<>();
List<JSONObject> jobajList = (List<JSONObject>) jsonObject.get(databaseName);
if (jobajList != null && jobajList.size() > 0) {
for (JSONObject jobj : jobajList) {
List<String> colList = (List<String>) jobj.get(collection);
if (colList != null && colList.size() > 0) {
for (String col : colList) {
fieldsToUpdate.add(col);
}
}
}
}
return fieldsToUpdate;
}
}
qida-dbmigrate-master/dbmigrate/src/main/java/com/qida/MongoDemo.java 0 → 100644
package com.qida;
import com.alibaba.fastjson.JSONObject;
import com.mongodb.*;
import com.mongodb.client.*;
import com.qida.mongoUtils.MongoDBUtil;
import org.bson.Document;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.*;
public class MongoDemo {
public static void main(String[] args) {
MongoClient src_mongoClient = null;
MongoClient dst_mongoClient = null;
try {
// 键盘输入需要迁移的数据库
String databaseName = "Survey";
// ------------------------ 开始操作数据库------------------------------------------------------------
// 数据源src
MongoCredential src_credential = MongoCredential.createCredential("qida", databaseName, "qida".toCharArray());
ServerAddress src_serverAddress = new ServerAddress("192.168.1.242", 3717);
src_mongoClient = new MongoClient(src_serverAddress, Arrays.asList(src_credential));
MongoDatabase src_mongoDatabase = src_mongoClient.getDatabase(databaseName);
MongoIterable<String> src_collectionNames = src_mongoDatabase.listCollectionNames();
// 迁移到的目标数据库dst
MongoCredential dst_credential = MongoCredential.createCredential("qida", databaseName, "qida".toCharArray());
ServerAddress dst_serverAddress = new ServerAddress("192.168.1.242", 3717);
dst_mongoClient = new MongoClient(dst_serverAddress, Arrays.asList(dst_credential));
MongoDatabase dst_mongoDatabase = dst_mongoClient.getDatabase(databaseName);
MongoIterable<String> dst_collectionNames = src_mongoDatabase.listCollectionNames();
// ----------------------------------------------------------------------------------------------------
MongoCollection<Document> collection = src_mongoDatabase.getCollection("EvaluationFeedback");
FindIterable<Document> documents = collection.find();
int i = 0;
for (Document document : documents) {
if (i > 10) {
break;
}
i++;
System.out.println(document);
}
} catch (Exception e) {
System.err.println("迁移时发生了错误！！！");
System.err.println(e.getClass().getName() + ": " + e.getMessage());
} finally {
if (src_mongoClient != null) {
src_mongoClient.close();
}
if (dst_mongoClient != null) {
dst_mongoClient.close();
}
}
}
}
qida-dbmigrate-master/dbmigrate/src/main/java/com/qida/MyApp.java 0 → 100644
package com.qida;
import com.alibaba.fastjson.JSONObject;
import com.mysql.cj.util.StringUtils;
import org.apache.commons.cli.*;
import org.apache.commons.dbcp2.BasicDataSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.sql.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;
public class MyApp {
private static final Logger logger = LoggerFactory.getLogger(App.class);
public static void main(String[] args) {
final Options options = new Options();
options.addRequiredOption("sh", "srchost", true, "The source mysql server host(including port), e.g. 192.168.56.101:3306.")
.addRequiredOption("dh", "dsthost", true, "The destination mysql server host(including port), e.g. 192.168.56.102:3307.")
.addRequiredOption("su", "srcuser", true, "The source mysql username.")
.addRequiredOption("du", "dstuser", true, "The destination mysql username.")
.addOption("sp", "srcpassword", true, "The source mysql password.")
.addOption("dp", "dstpassword", true, "The destination mysql password.");
final HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("QidaDataExtractor", options);
Connection fooConnection = null;
PreparedStatement fooStatement = null;
ResultSet fooResultSet = null;
PreparedStatement tableStatement = null;
ResultSet tableResultSet = null;
Connection barConnection = null;
Statement barStatement = null;
ResultSet barResultSet = null;
String insert_sql = "";
try {
final DefaultParser parser = new DefaultParser();
CommandLine cl = parser.parse(options, args);
final String sh = cl.getOptionValue("sh");
final String su = cl.getOptionValue("su");
final String sp = cl.getOptionValue("sp");
final String dh = cl.getOptionValue("dh");
final String du = cl.getOptionValue("du");
final String dp = cl.getOptionValue("dp");
logger.info("srchost= " + sh + ", dsthost= " + dh);
final BasicDataSource fooBds = new DataSource("jdbc:mysql://" + sh + "?useUnicode=true&characterEncoding=UTF-8", su, sp).getBds();
final BasicDataSource barBds = new DataSource("jdbc:mysql://" + dh + "?useUnicode=true&characterEncoding=UTF-8", du, dp).getBds();
// 键盘输入需要迁移的数据库
System.out.println("请输入需要迁移的数据库名称，重要的事情说三遍！！！");
System.out.println("请输入需要迁移的数据库名称，重要的事情说三遍！！！");
System.out.println("请输入需要迁移的数据库名称，重要的事情说三遍！！！");
Scanner sc = new Scanner(System.in);
String moveDatabase = sc.next();
// 开始查询库下面的表
fooConnection = fooBds.getConnection();
barConnection = barBds.getConnection();
barStatement = barConnection.createStatement();
String select_table_count = "SELECT COUNT(*) as taleCount FROM information_schema.TABLES WHERE table_schema = '" + moveDatabase + "' AND table_type = 'base table' AND table_name NOT LIKE '%_audit'";
String select_table = "SELECT table_name FROM information_schema.TABLES WHERE table_schema = '" + moveDatabase + "' AND table_type = 'base table' AND table_name NOT LIKE '%_audit%'";
tableStatement = fooConnection.prepareStatement(select_table_count);
tableResultSet = tableStatement.executeQuery();
int taleCount = 0;
while (tableResultSet.next()) {
taleCount = tableResultSet.getInt("taleCount");
}
if (taleCount <= 0) {
System.out.println("########## " + moveDatabase + "库的表数量是零，无需迁移！！！##########");
return;
}
System.out.println("########## " + moveDatabase + "库的迁移准备开始，共有" + taleCount + "张表需要迁移！！！##########");
long startTime = System.currentTimeMillis();
// 读取json文件
JSONObject jsonObject = getJsonObject();
// 查出所有表
tableStatement = fooConnection.prepareStatement(select_table);
tableResultSet = tableStatement.executeQuery();
List<String> tableList = new ArrayList<>();
while (tableResultSet.next()) {
String table = tableResultSet.getNString("table_name");
tableList.add(table);
}
// 开始每张表的迁移
int tableNo = 0;
for (String tableName : tableList) {
//for (int ii = 4; ii < tableList.size(); ii++) {
//String tableName = tableList.get(ii);
tableNo++;
System.out.println("########## 开始迁移第" + tableNo + "张" + tableName + "表的数据！！！##########");
/*if(!tableName.equals("rank_integral") && !tableName.equals("rank_score") && !tableName.equals("rank_time")) {
continue;
}*/
// 跳过这个表
if (tableName.equals("billing_info_log") || tableName.equals("company_channel_list_log")) {
continue;
}
// 首先删除dst各表的数据
if (moveDatabase.equals("clm-interact")) {
moveDatabase = "`clm-interact`";
}
String delete_data = "DELETE FROM " + moveDatabase + "." + tableName;
barStatement.executeUpdate(delete_data);
Thread.sleep(2000);
// 查出src各表的数据
String select_dataCount = "SELECT COUNT(*) dataCount FROM " + moveDatabase + "." + tableName;
fooStatement = fooConnection.prepareStatement(select_dataCount);
fooResultSet = fooStatement.executeQuery();
int dataCount = 0;
while (fooResultSet.next()) {
dataCount = fooResultSet.getInt("dataCount");
}
String select_data = "SELECT * FROM " + moveDatabase + "." + tableName;
fooStatement = fooConnection.prepareStatement(select_data);
fooResultSet = fooStatement.executeQuery();
// 查出每张表的字段
List<String> colList = new ArrayList<>();
ResultSetMetaData metaData = fooResultSet.getMetaData();
for (int i = 0; i < metaData.getColumnCount(); i++) {
colList.add(metaData.getColumnName((i + 1)));
}
int index = 0;
int num = 0;
String value = "";
while (fooResultSet.next()) {
index++;
value = value + "(";
for (int i = 0; i < colList.size(); i++) {
if (i == 0) {
if (isSpecialCol(jsonObject, moveDatabase, tableName, colList.get(i))) {
String val = fooResultSet.getString(colList.get(i));
// 特殊的字段特殊处理-------------------------------------------
if (val != null) {
val = dealSpecial(tableName + "." + colList.get(i), val);
}
// --------------------------------------------------------------
value = value + "\"" + val + "\",";
} else {
int id = fooResultSet.getInt(colList.get(i));
if (id > 0) {
value = value + "\"" + (id + 100000000) + "\", ";
} else if (id < 0) {
value = value + "\"" + (id - 100000000) + "\", ";
} else if (id == 0) {
value = value + "null" + ", ";
}
}
} else if (i > 0 && i < (colList.size() - 1)) {
if (isSpecialCol(jsonObject, moveDatabase, tableName, colList.get(i))) {
String val = fooResultSet.getString(colList.get(i));
// 特殊的字段特殊处理-------------------------------------------
if (val != null) {
val = dealSpecial(tableName + "." + colList.get(i), val);
}
// --------------------------------------------------------------
value = value + "\"" + val + "\",";
} else {
if (colList.get(i).contains("_id") || colList.get(i).contains("_ID")) {
int id = fooResultSet.getInt(colList.get(i));
if (id > 0) {
value = value + "\"" + (id + 100000000) + "\", ";
} else if (id < 0) {
value = value + "\"" + (id - 100000000) + "\", ";
} else if (id == 0) {
value = value + "null" + ", ";
}
} else {
String val = fooResultSet.getString(colList.get(i));
if (val != null) {
if (val.contains("\"")) {
val = val.replaceAll("\"", "\\\\\"");
}
if (val.endsWith("\\")) {
val = val + " ";
}
}
value = value + "\"" + val + "\",";
}
}
} else if (i > 0 && i == (colList.size() - 1)) {
if (isSpecialCol(jsonObject, moveDatabase, tableName, colList.get(i))) {
String val = fooResultSet.getString(colList.get(i));
// 特殊的字段特殊处理-------------------------------------------
if (val != null) {
val = dealSpecial(tableName + "." + colList.get(i), val);
}
// --------------------------------------------------------------
value = value + "\"" + val + "\"),";
} else {
if (colList.get(i).contains("_id") || colList.get(i).contains("_ID")) {
int id = fooResultSet.getInt(colList.get(i));
if (id > 0) {
value = value + "\"" + (id + 100000000) + "\"),";
} else if (id < 0) {
value = value + "\"" + (id - 100000000) + "\"),";
} else if (id == 0) {
value = value + "null" + "),";
}
} else {
String val = fooResultSet.getString(colList.get(i));
if (val != null) {
if (val.contains("\"")) {
val = val.replaceAll("\"", "\\\\\"");
System.out.println("val: " + val);
}
}
value = value + "\"" + val + "\"),";
}
}
}
}
// 每30条记录插入一次
if (dataCount > 0 && dataCount <= 30 && index == dataCount) {
value = value.replaceAll("\"null\"", "null");
value = value.substring(0, value.length() - 1);
insert_sql = "INSERT INTO " + moveDatabase + "." + tableName + " VALUES" + value;
barStatement.executeUpdate(insert_sql);
System.out.println("########## 已插入" + tableName + "的" + index + "条的数据插入，" + "还剩下" + (dataCount - index) + "条的数据待插入！！！##########");
} else if (dataCount > 30 && index % 30 == 0) {
value = value.replaceAll("\"null\"", "null");
value = value.substring(0, value.length() - 1);
insert_sql = "INSERT INTO " + moveDatabase + "." + tableName + " VALUES" + value;
barStatement.executeUpdate(insert_sql);
num++;
value = "";
System.out.println("########## 已插入" + tableName + "的" + index + "条的数据插入，" + "还剩下" + (dataCount - index) + "条的数据待插入！！！##########");
} else if (dataCount > 30 && index > 30 && (dataCount - 30 * num) == index % (30 * num)) {
value = value.replaceAll("\"null\"", "null");
value = value.substring(0, value.length() - 1);
insert_sql = "INSERT INTO " + moveDatabase + "." + tableName + " VALUES" + value;
barStatement.executeUpdate(insert_sql);
System.out.println("########## 已插入" + tableName + "的" + +index + "条的数据，" + "还剩下" + (dataCount - index) + "条的数据待插入！！！##########");
}
}
long endTime = System.currentTimeMillis();
System.out.println("########## 已完成第" + tableNo + "张表" + tableName + "的迁移，还剩下" + (tableList.size() - tableNo) + "张表的迁移，已使用时长" + ((double) ((endTime - startTime) / 1000)) + "秒！！！##########");
}
System.out.println("###########################################");
System.out.println("########## 数据库迁移已完成！！！##########");
System.out.println(getTip());
} catch (SQLException e) {
logger.error("Exception while executing query", e);
System.out.println("插入数据时出现insert语句错误： " + insert_sql);
} catch (ParseException e) {
logger.error("Exception while parsing args", e);
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
if (null != tableResultSet) {
tableResultSet.close();
}
if (null != fooResultSet) {
fooResultSet.close();
}
if (null != tableStatement) {
tableStatement.close();
}
if (null != fooStatement) {
fooStatement.close();
}
if (null != fooConnection) {
fooConnection.close();
}
if (null != barResultSet) {
barResultSet.close();
}
if (null != barStatement) {
barStatement.close();
}
if (null != barConnection) {
barConnection.close();
}
} catch (Exception e) {
logger.error("Exception while wrapping up", e);
}
}
}
// 特殊列的处理方法
private static boolean isSpecialCol(JSONObject jsonObject, String database, String table, String column) {
List<JSONObject> jobajList = (List<JSONObject>) jsonObject.get(database);
if (jobajList != null && jobajList.size() > 0) {
for (JSONObject jobj : jobajList) {
List<String> colList = (List<String>) jobj.get(table);
if (colList != null && colList.size() > 0) {
for (String col : colList) {
if (col.equalsIgnoreCase(column)) {
return true;
}
}
}
}
}
return false;
}
// --------------以下是处理有些表中特别扑街的字段-------------------------------------------------------------------
private static String dealSpecial(String specialCol, String val) {
switch (specialCol) {
// 处理很多如"101,102,103,104"每个加一亿的问题
case "ext_learn_group_thread.image_ids":
val = dealIds(val);
break;
case "org_company_config.agent_id":
val = dealAgentId(val);
break;
// 处理CompanyId为0的状况
case "billing_info.CompanyId":
val = (Integer.parseInt(val) + 100000000) + "";
break;
case "billing_info_log.OperatorCasId":
val = dealId(val);
break;
case "billing_info_log.CompanyId":
val = dealId(val);
break;
case "channel_info.ChannelId":
val = dealId(val);
break;
case "channel_info.TeacherCasId":
val = dealId(val);
break;
case "channel_info.GroupIdList":
val = dealIds2(val);
break;
case "channel_info.DepartmentIdList":
val = dealIds2(val);
break;
case "channel_info.UserCasIdList":
val = dealIds2(val);
break;
case "channel_polyv_summary.ChannelId":
val = dealId(val);
break;
case "train_task.logo_id":
long logoId = Long.parseLong(val);
if (logoId < 1231313213123L) {
val = logoId + 100000000 + "";
}
break;
case "company_channel_list.CompanyId":
val = dealId(val);
break;
case "company_channel_list.ChannelId":
val = dealId(val);
break;
case "company_channel_list_delete_from_polyv.CompanyId":
val = dealId(val);
break;
case "company_channel_list_delete_from_polyv.ChannelId":
val = dealId(val);
break;
case "company_last_create_channel_time.CompanyId":
val = dealId(val);
break;
case "company_polyv_account.CompanyId":
val = dealId(val);
break;
case "company_polyv_account.MainPolyvUserId":
val = dealId(val);
break;
case "exceptional_channel.PolyvUserId":
val = dealId(val);
break;
case "exceptional_channel.CompanyId":
val = dealId(val);
break;
case "exceptional_channel.ChannelId":
val = dealId(val);
break;
case "exceptional_channel.OperatorCasId":
val = dealId(val);
break;
case "recharge_minutes_log.CompanyId":
val = dealId(val);
break;
case "recharge_minutes_log.OperatorCasId":
val = dealId(val);
break;
case "settle_minutes_log.CompanyId":
val = dealId(val);
break;
case "settle_minutes_log.ChannelId":
val = dealId(val);
break;
case "settle_minutes_log.OperatorCasId":
val = dealId(val);
break;
case "ofid.id":
val = dealId(val);
break;
case "ofmucconversationlog.body":
val = dealBody(val);
break;
case "ofmucconversationlog.id":
val = dealId(val);
break;
case "ofmucroom.serviceID":
val = dealId(val);
break;
case "ofmucroom.roomID":
val = dealId(val);
break;
case "ofoffline.messageID":
val = dealId(val);
break;
case "ofsecurityauditlog.msgID":
val = dealId(val);
break;
case "srv_survey_paper.logo_id":
long logoId2 = Long.parseLong(val);
if (logoId2 < 1231313213123L) {
val = logoId2 + 100000000 + "";
}
break;
case "sys_menu.parent_ids":
int id = Integer.parseInt(val);
if (id == 0) {
val = id + "";
} else {
val = (id + 100000000) + "";
}
break;
case "sys_role_menu.update_date":
if (val.equals("0000-00-00 00:00:00")) {
val = null;
}
break;
case "sys_menu.target":
int target = Integer.parseInt(val);
if(target > 0 && target != 100) {
val = (target + 100000000) + "";
}
break;
case "qida_resource_authorization.library":
int library = Integer.parseInt(val);
if(library != 0) {
val = (library + 100000000) + "";
}
break;
case "qida_resource_thirdparty.id":
int id2 = Integer.parseInt(val);
if(id2 != 0) {
val = (id2 + 100000000) + "";
}
break;
case "qida_course_info.library":
int library2 = Integer.parseInt(val);
if(library2 != 0) {
val = (library2 + 100000000) + "";
}
break;
case "org_department.parent_id":
int id3 = Integer.parseInt(val);
if (id3 == 0) {
val = id3 + "";
} else {
val = (id3 + 100000000) + "";
}
break;
default:
break;
}
return val;
}
// 处理ofmucconversationlog.body这个字段的id加一亿
private static String dealBody(String val) {
JSONObject jsonObject = JSONObject.parseObject(val);
JSONObject bodyContext = (JSONObject) jsonObject.get("bodyContext");
JSONObject fluentPut = null;
if (jsonObject.get("messageNo") != null) {
Integer liveId = (Integer) bodyContext.get("liveId");
if (liveId != null) {
liveId = liveId + 100000000;
}
Integer userId = (Integer) bodyContext.get("userId");
if (userId != null) {
userId = userId + 100000000;
}
fluentPut = bodyContext.fluentPut("liveId", liveId).fluentPut("userId", userId);
} else {
String liveId = (String) bodyContext.get("liveId");
if (liveId != null) {
liveId = (Integer.parseInt(liveId) + 100000000) + "";
}
String userId = (String) bodyContext.get("userId");
if (userId != null) {
userId = (Integer.parseInt(userId) + 100000000) + "";
}
fluentPut = bodyContext.fluentPut("liveId", liveId).fluentPut("userId", userId);
}
String result = jsonObject.fluentPut("bodyContext", fluentPut).toString();
result = result.replaceAll("\"", "\\\\\"");
return result;
}
// 处理org_company_config.agent_id的空串空格的问题
private static String dealAgentId(String val) {
if (!val.trim().equals("")) {
int id = Integer.parseInt(val.trim());
if (val.startsWith(" ")) {
val = " " + (id + 100000000);
} else {
val = (id + 100000000) + "";
}
}
return val;
}
// 处理很多如"101,102,103,104"每个加一亿的问题
private static String dealIds(String val) {
String result = "";
String[] split = val.split(",");
for (String s : split) {
if (s != null && !s.trim().equals("")) {
int i = Integer.parseInt(s);
if (i >= 0) {
i = i + 100000000;
} else {
i = i - 100000000;
}
result = result + i + ",";
} else {
result = result + s + ",";
}
}
if (!val.endsWith(",")) {
result = result.substring(0, result.length() - 1);
}
return result;
}
private static String dealIds2(String val) {
if (val.equals("null")) {
return val;
}
val = val.substring(1, val.length() - 1);
String result = "";
String[] split = val.split(",");
for (
String s : split) {
if (s != null && !s.trim().equals("")) {
int i = Integer.parseInt(s);
if (i >= 0) {
i = i + 100000000;
} else {
i = i - 100000000;
}
result = result + i + ",";
} else {
result = result + s + ",";
}
}
if (!val.endsWith(",")) {
result = result.substring(0, result.length() - 1);
}
result = "[" + result + "]";
return result;
}
// 处理类型xxxxxId的值方法
private static String dealId(String val) {
int operatorCasId = Integer.parseInt(val);
if (operatorCasId >= 0) {
val = operatorCasId + 100000000 + "";
} else {
val = operatorCasId - 100000000 + "";
}
return val;
}
// -----------------------------------------------------------------------------------------------------------------
private static JSONObject getJsonObject() throws Exception {
String path = MyApp.class.getClassLoader().getResource("sc.json").getPath();
String jsonStr = "";
File jsonFile = new File(path);
Reader reader = new InputStreamReader(new FileInputStream(jsonFile), "utf-8");
int ch = 0;
StringBuffer sb = new StringBuffer();
while ((ch = reader.read()) != -1) {
sb.append((char) ch);
}
reader.close();
jsonStr = sb.toString();
JSONObject jsonObject = JSONObject.parseObject(jsonStr);
return jsonObject;
}
private static String getTip() {
String s = "┏┳━━━━━━━━━━┓\n" +
"┃┃██████████████████┃\n" +
"┃┃█████████████┏━┓█┃\n" +
"┃┃█████████████┃－┃█┃\n" +
"┣┫█████████████┃乾┃█┃\n" +
"┃┃█████████████┃坤┃█┃\n" +
"┃┃█████████████┃大┃█┃\n" +
"┃┃█████████████┃挪┃█┃\n" +
"┃┃█████████████┃移┃█┃\n" +
"┃┃█████████████┃－┃█┃\n" +
"┣┫█████████████┗━┛█┃\n" +
"┃┃██████████████████┃\n" +
"┃┃██████████████████┃\n" +
"┃┃██████████████████┃\n" +
"┃┃█████████ 定价５毛┃\n" +
"┗┻━━━━━━━━━━━┛\n" +
"欲要迁移成功，必须练成此神功";
return s;
}
}
qida-dbmigrate-master/dbmigrate/src/main/java/com/qida/MyApp2.java 0 → 100644
package com.qida;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.cli.*;
import org.apache.commons.dbcp2.BasicDataSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;
public class MyApp2 {
private static final Logger logger = LoggerFactory.getLogger(App.class);
public static void main(String[] args) {
final Options options = new Options();
options.addRequiredOption("sh", "srchost", true, "The source mysql server host(including port), e.g. 192.168.56.101:3306.")
.addRequiredOption("dh", "dsthost", true, "The destination mysql server host(including port), e.g. 192.168.56.102:3307.")
.addRequiredOption("su", "srcuser", true, "The source mysql username.")
.addRequiredOption("du", "dstuser", true, "The destination mysql username.")
.addOption("sp", "srcpassword", true, "The source mysql password.")
.addOption("dp", "dstpassword", true, "The destination mysql password.");
final HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("QidaDataExtractor", options);
Connection fooConnection = null;
PreparedStatement fooStatement = null;
ResultSet fooResultSet = null;
PreparedStatement tableStatement = null;
ResultSet tableResultSet = null;
Connection barConnection = null;
Statement barStatement = null;
ResultSet barResultSet = null;
try {
final DefaultParser parser = new DefaultParser();
CommandLine cl = parser.parse(options, args);
final String sh = cl.getOptionValue("sh");
final String su = cl.getOptionValue("su");
final String sp = cl.getOptionValue("sp");
final String dh = cl.getOptionValue("dh");
final String du = cl.getOptionValue("du");
final String dp = cl.getOptionValue("dp");
logger.info("srchost= " + sh + ", dsthost= " + dh);
final BasicDataSource fooBds = new DataSource("jdbc:mysql://" + sh + "?useUnicode=true&characterEncoding=UTF-8", su, sp).getBds();
final BasicDataSource barBds = new DataSource("jdbc:mysql://" + dh + "?useUnicode=true&characterEncoding=UTF-8", du, dp).getBds();
// 键盘输入需要迁移的数据库
System.out.println("请输入需要修改的数据库名称，重要的事情说三遍！！！");
System.out.println("请输入需要修改的数据库名称，重要的事情说三遍！！！");
System.out.println("请输入需要修改的数据库名称，重要的事情说三遍！！！");
Scanner sc = new Scanner(System.in);
String moveDatabase = sc.next();
fooConnection = fooBds.getConnection();
barConnection = barBds.getConnection();
barStatement = barConnection.createStatement();
String select_table_count = "SELECT COUNT(*) as taleCount FROM information_schema.TABLES WHERE table_schema = '" + moveDatabase + "' AND table_type = 'base table' AND table_name NOT LIKE '%_audit'";
String select_table = "SELECT table_name FROM information_schema.TABLES WHERE table_schema = '" + moveDatabase + "' AND table_type = 'base table'";
tableStatement = fooConnection.prepareStatement(select_table_count);
tableResultSet = tableStatement.executeQuery();
int taleCount = 0;
while (tableResultSet.next()) {
taleCount = tableResultSet.getInt("taleCount");
}
if (taleCount <= 0) {
System.out.println("########## " + moveDatabase + "库的表数量是零，无需迁移！！！##########");
return;
}
System.out.println("########## " + moveDatabase + "库的迁移准备开始，共有" + taleCount + "张表需要迁移！！！##########");
// 查出所有表
tableStatement = fooConnection.prepareStatement(select_table);
tableResultSet = tableStatement.executeQuery();
List<String> tableList = new ArrayList<>();
while (tableResultSet.next()) {
String table = tableResultSet.getNString("table_name");
tableList.add(table);
}
for (String tableName : tableList) {
if (tableName.contains("z_dst_")) {
continue;
}
String newTableName = "z_src_" + tableName;
String alert_sql = "alter table " + moveDatabase + "." + tableName + " rename " + newTableName + ";";
System.out.println(alert_sql);
}
/*for (String tableName : tableList) {
if(tableName.contains("z_src_")) {
String newTableName = tableName.replaceAll("z_src_","");
String alert_sql = "alter table " + moveDatabase + "." + tableName +" rename " + newTableName + ";";
System.out.println(alert_sql);
}
}*/
} catch (SQLException e) {
logger.error("Exception while executing query", e);
} catch (ParseException e) {
logger.error("Exception while parsing args", e);
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
if (null != tableResultSet) {
tableResultSet.close();
}
if (null != fooResultSet) {
fooResultSet.close();
}
if (null != tableStatement) {
tableStatement.close();
}
if (null != fooStatement) {
fooStatement.close();
}
if (null != fooConnection) {
fooConnection.close();
}
if (null != barResultSet) {
barResultSet.close();
}
if (null != barStatement) {
barStatement.close();
}
if (null != barConnection) {
barConnection.close();
}
} catch (Exception e) {
logger.error("Exception while wrapping up", e);
}
}
}
}
qida-dbmigrate-master/dbmigrate/src/main/java/com/qida/mongoUtils/MongoDBUtil.java 0 → 100644
package com.qida.mongoUtils;
import cn.hutool.core.collection.CollUtil;
import org.bson.Document;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
/**
* 此类是mongodb迁移时特别要处理字段的类
*/
public class MongoDBUtil {
private final static long yiyi = 100000000;
public static List<Document> dealDoc(String colName, List<Document> documents, List<String> fieldList) {
switch (colName) {
// Survey数据库
case "Survey.EvaluationFeedback":
documents = dealEvaluationFeedback(documents, fieldList);
break;
case "Survey.LecturerEvaluationAct":
documents = dealLecturerEvaluationAct(documents, fieldList);
break;
case "Survey.LecturerEvaluationPaper":
documents = dealCommon(documents, fieldList);
break;
case "Survey.PaperOfEvaluationQuestionReport":
documents = dealCommon(documents, fieldList);
break;
case "Survey.SurveyAct":
documents = dealSurveyAct(documents, fieldList);
break;
case "Survey.SurveyFeedback":
documents = dealSurveyFeedback(documents, fieldList);
break;
case "Survey.SurveyPaper":
documents = dealCommon(documents, fieldList);
break;
// clm-custom数据库
case "clm-custom.sundanceLecturer":
documents = dealCommon(documents, fieldList);
break;
// patch_Examination数据库
case "patch_Examination.ExamPaperStaticData":
documents = dealExamPaperStaticData(documents, fieldList);
break;
case "patch_Examination.ExamTrackingUserAnswerData":
documents = dealExamTrackingUserAnswerData(documents, fieldList);
break;
// patch_lmstracking数据库
case "patch_lmstracking.lMSItemAttemp":
documents = dealCommon(documents, fieldList);
break;
case "patch_lmstracking.lMSSTDPerformance":
documents = dealLMSSTDPerformance(documents, fieldList);
break;
case "patch_lmstracking.lMSScormPerformance":
documents = dealLMSScormPerformance(documents, fieldList);
break;
// tenant数据库
case "tenant.Individuation":
documents = dealIndividuation(documents, fieldList);
break;
default:
break;
}
return documents;
}
// -----------------------------------------------------------------------------------------------------------------
// 处理id方法
private static Long dealId(Document document, String key) {
Long id = (Long) document.get(key);
if (id != null) {
if (id > 0) {
id = id + yiyi;
} else if (id < 0) {
id = id - yiyi;
}
}
return id;
}
// -----------------------------------------------------------------------------------------------------------------
// 这是只处理外层字段的方法
private static List<Document> dealCommon(List<Document> documents, List<String> fieldList) {
for (Document document : documents) {
for (String field : fieldList) {
if (!field.contains("_inline")) {
if(document.get(field) != null) {
document.put(field, dealId(document, field));
}
}
}
}
return documents;
}
// 处理Survey.EvaluationFeedback字段方法
private static List<Document> dealEvaluationFeedback(List<Document> documents, List<String> fieldList) {
for (Document document : documents) {
for (String field : fieldList) {
if (!field.contains("_inline")) {
if(document.get(field) != null) {
document.put(field, dealId(document, field));
}
} else if (field.contains("feedbackAnswers")) {
List<Document> documentList = (List<Document>) document.get("feedbackAnswers");
if (CollUtil.isNotEmpty(documentList)) {
for (Document doc : documentList) {
if(doc.get("paperId") != null) {
doc.put("paperId", dealId(doc, "paperId"));
}
}
}
document.put("feedbackAnswers", documentList);
}
}
}
return documents;
}
// 处理Survey.LecturerEvaluationAct字段方法
private static List<Document> dealLecturerEvaluationAct(List<Document> documents, List<String> fieldList) {
for (Document document : documents) {
for (String field : fieldList) {
if (!field.contains("_inline")) {
if(document.get(field) != null) {
document.put(field, dealId(document, field));
}
} else if (field.contains("paper")) {
Document paper = (Document) document.get("paper");
if(paper.get("lecturerId") != null) {
paper.put("lecturerId", dealId(paper, "lecturerId"));
}
if(paper.get("companyId") != null) {
paper.put("companyId", dealId(paper, "companyId"));
}
if(paper.get("creatorId") != null) {
paper.put("creatorId", dealId(paper, "creatorId"));
}
}
}
}
return documents;
}
// 处理Survey.SurveyAct字段方法
private static List<Document> dealSurveyAct(List<Document> documents, List<String> fieldList) {
for (Document document : documents) {
for (String field : fieldList) {
if (!field.contains("_inline")) {
if(document.get(field) != null) {
document.put(field, dealId(document, field));
}
} else if (field.contains("paper")) {
Document paper = (Document) document.get("paper");
if(paper.get("logoId") != null) {
paper.put("logoId", dealId(paper, "logoId"));
}
if(paper.get("companyId") != null) {
paper.put("companyId", dealId(paper, "companyId"));
}
if(paper.get("creatorId") != null) {
paper.put("creatorId", dealId(paper, "creatorId"));
}
}
}
}
return documents;
}
// 处理Survey.SurveyFeedback字段方法
private static List<Document> dealSurveyFeedback(List<Document> documents, List<String> fieldList) {
for (Document document : documents) {
for (String field : fieldList) {
if (!field.contains("_inline")) {
if(document.get(field) != null) {
document.put(field, dealId(document, field));
}
} else if (field.contains("feedbackAnswers")) {
List<Document> documentList = (List<Document>) document.get("feedbackAnswers");
if (CollUtil.isNotEmpty(documentList)) {
for (Document doc : documentList) {
if(doc.get("paperId") != null) {
doc.put("paperId", dealId(doc, "paperId"));
}
}
}
document.put("feedbackAnswers", documentList);
}
}
}
return documents;
}
// 处理patch_Examination.ExamPaperStaticData字段方法
private static List<Document> dealExamPaperStaticData(List<Document> documents, List<String> fieldList) {
for (Document document : documents) {
for (String field : fieldList) {
if (!field.contains("_inline")) {
if(document.get(field) != null) {
document.put(field, dealId(document, field));
}
} else if (field.contains("headings")) {
List<Document> documentList = (List<Document>) document.get("headings");
if (CollUtil.isNotEmpty(documentList)) {
for (Document doc : documentList) {
if(doc.get("paperId") != null) {
doc.put("paperId", dealId(doc, "paperId"));
}
if(doc.get("companyId") != null) {
doc.put("companyId", dealId(doc, "companyId"));
}
if(doc.get("creatorId") != null) {
doc.put("creatorId", dealId(doc, "creatorId"));
}
if(doc.get("updaterId") != null) {
doc.put("updaterId", dealId(doc, "updaterId"));
}
List<Document> questions = (List<Document>) doc.get("questions");
if (CollUtil.isNotEmpty(questions)) {
for (Document doc1 : questions) {
if(doc1.get("categoryId") != null) {
doc1.put("categoryId", dealId(doc1, "categoryId"));
}
if(doc1.get("companyId") != null) {
doc1.put("companyId", dealId(doc1, "companyId"));
}
if(doc1.get("creatorId") != null) {
doc1.put("creatorId", dealId(doc1, "creatorId"));
}
if(doc1.get("updaterId") != null) {
doc1.put("updaterId", dealId(doc1, "updaterId"));
}
List<Document> answers = (List<Document>) doc1.get("answers");
if (CollUtil.isNotEmpty(answers)) {
for (Document doc2 : answers) {
if(doc2.get("questionId") != null) {
doc2.put("questionId", dealId(doc2, "questionId"));
}
if(doc2.get("companyId") != null) {
doc2.put("companyId", dealId(doc2, "companyId"));
}
if(doc2.get("creatorId") != null) {
doc2.put("creatorId", dealId(doc2, "creatorId"));
}
if(doc2.get("updaterId") != null) {
doc2.put("updaterId", dealId(doc2, "updaterId"));
}
}
}
doc1.put("answers", answers);
}
}
doc.put("questions", questions);
}
}
document.put("headings", documentList);
}
}
}
return documents;
}
// 处理patch_Examination.ExamTrackingUserAnswerData字段方法
private static List<Document> dealExamTrackingUserAnswerData(List<Document> documents, List<String> fieldList) {
for (Document document : documents) {
for (String field : fieldList) {
if (!field.contains("_inline")) {
if(document.get(field) != null) {
document.put(field, dealId(document, field));
}
} else if (field.contains("answers")) {
Document document1 = (Document) document.get("answers");
if (document1 != null) {
Set<String> keySet = document1.keySet();
if (CollUtil.isNotEmpty(keySet)) {
for (String key : keySet) {
Document document2 = (Document) document1.get(key);
if(document2.get("questionId") != null) {
document2.put("questionId", dealId(document2, "questionId"));
}
List<Long> inputAnswersList = (List<Long>) document2.get("inputAnswers");
List<Long> newInputAnswersList = new ArrayList<>();
if (CollUtil.isNotEmpty(inputAnswersList)) {
for (Long item : inputAnswersList) {
newInputAnswersList.add(item + yiyi);
}
document2.put("inputAnswers", newInputAnswersList);
}
}
}
}
} else if (field.contains("paperLayout")) {
Document document1 = (Document) document.get("paperLayout");
if (document1 != null) {
if(document1.get("objectId") != null) {
document1.put("objectId", dealId(document1, "objectId"));
}
Document document2 = (Document) document.get("subDisplaySeqMap");
if (document2 != null) {
Set<String> keySet = document2.keySet();
if (CollUtil.isNotEmpty(keySet)) {
for (String key : keySet) {
Document document3 = (Document) document2.get(key);
if(document3.get("objectId") != null) {
document3.put("objectId", dealId(document3, "objectId"));
}
Document document4 = (Document) document3.get("subDisplaySeqMap");
if (document4 != null) {
Set<String> keySet1 = document4.keySet();
if (CollUtil.isNotEmpty(keySet1)) {
for (String key1 : keySet1) {
Document document5 = (Document) document4.get(key1);
if (document5 != null) {
if(document5.get("objectId") != null) {
document5.put("objectId", dealId(document5, "objectId"));
}
}
}
}
}
}
}
}
}
}
}
}
return documents;
}
// 处理patch_lmstracking.lMSSTDPerformance字段方法
private static List<Document> dealLMSSTDPerformance(List<Document> documents, List<String> fieldList) {
for (Document document : documents) {
for (String field : fieldList) {
if (!field.contains("_inline")) {
if(document.get(field) != null) {
document.put(field, dealId(document, field));
}
} else if (field.contains("itemPerformances")) {
List<Document> documentList = (List<Document>) document.get("itemPerformances");
if (CollUtil.isNotEmpty(documentList)) {
for (Document document1 : documentList) {
if(document1.get("userId") != null) {
document1.put("userId", dealId(document1, "userId"));
}
if(document1.get("crsId") != null) {
document1.put("crsId", dealId(document1, "crsId"));
}
}
}
}
}
}
return documents;
}
// 处理patch_lmstracking.lMSScormPerformance字段方法
private static List<Document> dealLMSScormPerformance(List<Document> documents, List<String> fieldList) {
for (Document document : documents) {
for (String field : fieldList) {
if (!field.contains("_inline")) {
if(document.get(field) != null) {
document.put(field, dealId(document, field));
}
} else if (field.contains("itemPerformances")) {
List<Document> documentList = (List<Document>) document.get("itemPerformances");
if (CollUtil.isNotEmpty(documentList)) {
for (Document document1 : documentList) {
if(document1.get("userId") != null) {
document1.put("userId", dealId(document1, "userId"));
}
if(document1.get("crsId") != null) {
document1.put("crsId", dealId(document1, "crsId"));
}
}
}
}
}
}
return documents;
}
// 处理tenant.Individuation字段方法
private static List<Document> dealIndividuation(List<Document> documents, List<String> fieldList) {
for (Document document : documents) {
for (String field : fieldList) {
if (!field.contains("_inline")) {
if(document.get(field) != null) {
document.put(field, dealId(document, field));
}
} else if (field.contains("corporateIdentity")) {
Document document1 = (Document) document.get("corporateIdentity");
if(document1.get("loginImgId") != null) {
document1.put("loginImgId", dealId(document1, "loginImgId"));
}
if(document1.get("pageImgId") != null) {
document1.put("pageImgId", dealId(document1, "pageImgId"));
}
if(document1.get("userPortraitImg") != null) {
document1.put("userPortraitImg", dealId(document1, "userPortraitImg"));
}
document.put("corporateIdentity", document1);
}
}
}
return documents;
}
}
qida-dbmigrate-master/dbmigrate/src/main/resources/logback.xml 0 → 100644
<!-- Kindly note that this file should be put under the application classpath, e.g. in Tomcat the "${catalina.base}/lib/". Reference https://logback.qos.ch/manual/configuration.html. -->
<!-- Variable in "logback.xml" reference http://logback.qos.ch/manual/configuration.html#definingProps. -->
<configuration scan="true" scanPeriod="30 seconds">
<appender name="Console" class="ch.qos.logback.core.ConsoleAppender" withJansi="true">
<layout class="ch.qos.logback.classic.PatternLayout">
<pattern>%date %level %class::%method @line#%line [%thread] %message%n</pattern>
</layout>
</appender>
<appender name="RollingFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>debug</level>
</filter>
<file>${logs.home:-./}app.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>${logs.home:-./}app-%d{yyyy-MM-dd}.log.%i</fileNamePattern>
<maxFileSize>64MB</maxFileSize>
<totalSizeCap>16GB</totalSizeCap>
<maxHistory>64</maxHistory>
</rollingPolicy>
<append>true</append>
<layout class="ch.qos.logback.classic.PatternLayout">
<pattern>%date %level %class::%method @line#%line [%thread] %message%n</pattern>
</layout>
</appender>
<!-- Level filters of individual appenders below are specified respectively. -->
<root>
<appender-ref ref="RollingFile"/>
<appender-ref ref="Console"/>
</root>
</configuration>
qida-dbmigrate-master/dbmigrate/src/main/resources/mongodb.json 0 → 100644
{
"Survey": [
{
"EvaluationFeedback": [
"userId",
"actId",
"companyId",
"feedbackAnswers_inline"
]
},
{
"LecturerEvaluationAct": [
"companyId",
"lecturerId",
"paperId",
"creatorId",
"updaterId",
"paper_inline"
]
},
{
"LecturerEvaluationPaper": [
"lecturerId",
"trainId",
"companyId",
"creatorId",
"updaterId"
]
},
{
"PaperOfEvaluationQuestionReport": [
"lecturerId"
]
},
{
"SurveyAct": [
"companyId",
"paperId",
"creatorId",
"releaserId",
"updaterId",
"paper_inline"
]
},
{
"SurveyFeedback": [
"actId",
"companyId",
"feedbackAnswers_inline"
]
},
{
"SurveyPaper": [
"logoId",
"companyId",
"creatorId"
]
}
],
"clm-custom": [
{
"sundanceLecturer": [
"avatar"
]
}
],
"patch_Examination": [
{
"ExamPaperStaticData": [
"categoryId",
"companyId",
"creatorId",
"updaterId",
"headings_inline"
]
},
{
"ExamTrackingUserAnswerData": [
"paperId",
"answers_inline",
"paperLayout_inline"
]
}
],
"patch_lmstracking": [
{
"lMSItemAttemp": [
"itemId",
"userId",
"crsId"
]
},
{
"lMSSTDPerformance": [
"userId",
"crsId",
"lessonLocation",
"itemPerformances_inline"
]
},
{
"lMSScormPerformance": [
"userId",
"crsId",
"lessonLocation",
"itemPerformances_inline"
]
}
],
"tenant": [
{
"Individuation": [
"companyId",
"corporateIdentity_inline"
]
}
]
}
\ No newline at end of file
qida-dbmigrate-master/dbmigrate/src/main/resources/sc.json 0 → 100644
{
"qida_cas_platform": [
{
"cas_user_oauth": [
"app_id",
"open_id"
]
}
],
"clm_base_platform": [
{
"org_department": [
"parent_id"
]
},
{
"ext_learn_group_thread": [
"image_ids"
]
},
{
"org_company_config": [
"corp_id",
"agent_id"
]
},
{
"org_user": [
"other_user_id"
]
},
{
"org_user_menu": [
"parent_ids"
]
},
{
"sal_pros_customer": [
"JOB_ID",
"CMP_SCALE_ID"
]
},
{
"sys_menu": [
"parent_ids",
"target"
]
},
{
"sys_role_menu": [
"update_date"
]
}
],
"clm_live": [
{
"billing_info": [
"CompanyId"
]
},
{
"billing_info_log": [
"OperatorCasId",
"CompanyId",
"__#alibaba_rds_row_id#__"
]
},
{
"channel_info": [
"ChannelId",
"TeacherCasId",
"GroupIdList",
"DepartmentIdList",
"UserCasIdList"
]
},
{
"channel_polyv_summary": [
"ChannelId",
"__#alibaba_rds_row_id#__"
]
},
{
"common_config": [
"ConfigKey"
]
},
{
"company_channel_list": [
"CompanyId",
"ChannelId"
]
},
{
"company_channel_list_delete_from_polyv": [
"CompanyId",
"ChannelId"
]
},
{
"company_last_create_channel_time": [
"CompanyId"
]
},
{
"company_polyv_account": [
"CompanyId",
"MainPolyvUserId"
]
},
{
"exceptional_channel": [
"PolyvUserId",
"CompanyId",
"ChannelId",
"OperatorCasId",
"__#alibaba_rds_row_id#__"
]
},
{
"recharge_minutes_log": [
"CompanyId",
"OperatorCasId",
"__#alibaba_rds_row_id#__"
]
},
{
"settle_minutes_log": [
"CompanyId",
"ChannelId",
"OperatorCasId",
"__#alibaba_rds_row_id#__"
]
},
{
"polyv_account": [
"UserId"
]
}
],
"clm_offline": [
{
"train_task": [
"logo_id"
]
}
],
"clm_openfire": [
{
"ofid": [
"id"
]
},
{
"ofmucconversationlog": [
"body",
"id"
]
},
{
"ofmucroom": [
"serviceID",
"roomID"
]
},
{
"ofoffline": [
"username",
"messageID"
]
},
{
"ofpresence": [
"username"
]
},
{
"ofprivacylist": [
"username"
]
},
{
"ofprivate": [
"username"
]
},
{
"ofproperty": [
"name"
]
},
{
"ofpubsubaffiliation": [
"serviceID"
]
},
{
"ofpubsubdefaultconf": [
"serviceID"
]
},
{
"ofpubsubitem": [
"serviceID"
]
},
{
"ofpubsubnode": [
"serviceID"
]
},
{
"ofpubsubnodegroups": [
"serviceID"
]
},
{
"ofpubsubnodejids": [
"serviceID"
]
},
{
"ofremoteserverconf": [
"xmppDomain"
]
},
{
"ofsecurityauditlog": [
"msgID"
]
},
{
"ofuser": [
"username"
]
},
{
"ofuserprop": [
"username"
]
},
{
"ofversion": [
"name"
]
}
],
"clm_survey_platform": [
{
"srv_survey_paper": [
"logo_id"
]
}
],
"sys_app_push": [
{
"push_channel_client": [
"push_user_id",
"push_channel_id"
]
}
],
"sys_platform": [
{
"cloud_attachment_bucket_middle_item": [
"analysis_job_id",
"transcode_job_id"
]
},
{
"cloud_oss_company": [
"access_key_id"
]
},
{
"sys_app_version": [
"version_id",
"app_id",
"fix_id"
]
},
{
"sys_area": [
"area_pid",
"pid_ids",
"id"
]
},
{
"sys_area_pinyin": [
"ID",
"AREA_PID"
]
},
{
"sys_domain": [
"domain_pid",
"pid_ids"
]
},
{
"sys_feed_back": [
"app_id"
]
},
{
"sys_file_resource_temp": [
"__#alibaba_rds_row_id#__"
]
},
{
"sys_industry": [
"industry_pid",
"pid_ids"
]
}
],
"clm_resource_platform": [
{
"crs_course_info": [
"zbx_course_id"
]
},
{
"crs_course_item_scorm": [
"sco_idetifier",
"parent_idetifier"
]
},
{
"crs_course_material": [
"ext_identifer"
]
},
{
"qida_course_item_scorm": [
"sco_idetifier",
"parent_idetifier"
]
},
{
"qida_resource_authorization": [
"library"
]
},
{
"qida_resource_thirdparty": [
"id"
]
},
{
"": [
"library"
]
}
]
}
\ No newline at end of file
qida-dbmigrate-master/dbmigrate/src/test/java/com/qida/AppTest.java 0 → 100644
package com.qida;
import static org.junit.Assert.assertTrue;
import org.junit.Test;
/**
* Unit test for simple App.
*/
public class AppTest
{
/**
* Rigorous Test :-)
*/
@Test
public void shouldAnswerWithTrue()
{
assertTrue( true );
}
}
qida-dbmigrate-master/dockerize/Dockerfile 0 → 100644
FROM mcr.microsoft.com/java/jre:8-zulu-alpine
RUN mkdir -p /app
RUN mkdir -p /app/dbmigrate_cache
VOLUME /app/dbmigrate_cache
COPY docker-entrypoint.sh /usr/local/bin/
RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh # backwards compat
# deliberately using "COPY" instead of "ADD", please prepare the source files locally
COPY dbmigrate.jar /app/dbmigrate.jar
COPY lib /app/lib
ENTRYPOINT ["docker-entrypoint.sh"]
qida-dbmigrate-master/dockerize/docker-entrypoint.sh 0 → 100644
#!/bin/sh
# There's no bash support in image "mcr.microsoft.com/java/jre:8-zulu-alpine".
#echo Your container args are: "$@"
java -jar /app/dbmigrate.jar "$@"
qida-dbmigrate-master/dockerize/docker_build.sh 0 → 100644
#!/bin/bash
basedir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
cd $basedir && docker build -t qida_dbmigrate .
qida-dbmigrate-master/dockerize/docker_run_app.sh 0 → 100644
#!/bin/bash
# docker volume create shared_dbmigrate_cache
#cmd="docker run -d --mount 'type=volume,src=shared_dbmigrate_cache,dst=/app/dbmigrate_cache' --net=host dbmigrate:latest $@"
cmd="docker run -it --mount 'type=volume,src=shared_dbmigrate_cache,dst=/app/dbmigrate_cache' --net=host dbmigrate:latest $@"
eval $cmd
qida-dbmigrate-master/shortcut.sh 0 → 100644
#!/bin/bash
if [ $# -ne 1 ]; then
echo "Usage: $0 <operation code, e.g. \"create_subprojects\" or \"exec_dbmigrate\">"
exit 1
fi
basedir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
groupId=com.qida
archetypeArtifactId=maven-archetype-quickstart
archetypeVersion=1.4
dbmigrateId=dbmigrate
function create_subprojects () {
subprojects=( $dbmigrateId )
for i in "${subprojects[@]}"
do
if [[ ! -d $basedir/$i ]]; then
echo "Creating subproject $i"
cd $basedir && mvn -X archetype:generate -DgroupId=$groupId -DartifactId=$i -DarchetypeArtifactId=$archetypeArtifactId -DarchetypeVersion=$archetypeVersion -DinteractiveMode=false
fi
done
}
function package_dbmigrate() {
if [[ -d $basedir/$dbmigrateId ]]; then
echo "Packaging subproject $dbmigrateId"
# The property "logs.home" is used in "logback.xml".
cd $basedir/$dbmigrateId && mvn clean package -DskipTests
fi
}
function exec_dbmigrate() {
if [[ -d $basedir/$dbmigrateId ]]; then
echo "Executing subproject $dbmigrateId"
# The property "logs.home" is used in "logback.xml".
package_dbmigrate && cd $basedir/$dbmigrateId/target && java -Dlogs.home=$basedir/$dbmigrateId/logs/ -jar dbmigrate.jar
fi
}
operationCode=$1
$operationCode
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment