文本数据增量导入到mysql

时间:2023-03-09 09:43:20
文本数据增量导入到mysql

实现思路:
       实现Java读取TXT文件中的内容并存到内存,将内存中的数据和mysql 数据库里面某张表数据的字段做一个比较,如果比较内存中的数据在mysql 里存在则不做处理,如果不存在则将该数据插入mysql数据库中

步骤1、读取文本数据   给一个string 返回值
步骤2、查询mysql 表中数据  给一个String 返回值

步骤3  、内存中文本数据和读取的mysql 数据做比较

/**
* 实现读取文件信息
*
* @param fileName
* @return
*/
public static String readFileByLines(String fileName) {
String result = "";
file = new File(fileName);
String tempString = null;
BufferedReader reader = null;
try { reader = new BufferedReader(new FileReader(file));
while ((tempString = reader.readLine()) != null) {
result += tempString + "\n";
}
reader.close();
} catch (IOException e) {
e.printStackTrace();
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException e1) {
}
}
}
return result;
}
     public static String url = null;
public static String username = null;
public static String password = null;
public static Connection conn;
public static Statement stmt;
public static ResultSet rs;
public static String fileName;
public static String tempString = null; public static String PATH = "/dbconfig.properties";
private static Properties properties;
static {
try {
InputStream is = DBlUtils.class.getResourceAsStream(PATH);
properties = new Properties();
properties.load(is);
url = properties.getProperty("jdbc.url");
username = properties.getProperty("jdbc.username");
password = properties.getProperty("jdbc.password");
fileName = properties.getProperty("fileName");
System.out.println("fileName:" + fileName);
if (is != null)
is.close();
} catch (IOException e) { e.printStackTrace();
} }

查询mysql 数据库数据

/**
* 查询mysql 数据库数据,并获得内容
*
* @param sql
*/
public static String queryDatas(String sql) {
String queryDatas = null;
try {
conn = DriverManager.getConnection(url, username, password);
conn.setAutoCommit(false);
stmt = conn.prepareStatement("load data local infile '' " + "into table loadtest fields terminated by ','");
StringBuilder sb = new StringBuilder();
InputStream is = new ByteArrayInputStream(sb.toString().getBytes());
((com.mysql.jdbc.Statement) stmt).setLocalInfileInputStream(is);
ResultSet rs = stmt.executeQuery(sql);
ResultSetMetaData rsmd = rs.getMetaData();
int columnCount = rsmd.getColumnCount();
// 输出列名
for (int i = 1; i <= columnCount; i++) {
if (rsmd.getColumnName(i).startsWith("content")) {
System.out.print(rsmd.getColumnName(i));
}
}
System.out.println();
// 输出mysql数据
while (rs.next()) {
queryDatas = rs.getString(2);
}
} catch (Exception e) {
e.printStackTrace();
}
return queryDatas;
}

文本读取的数据和mysql查询的数据进行比较: