用shell脚本配合awk文本处理工具进行文件处理
由于项目是运行在linux环境下。需要统计文件中的记录金额总和等功能，于是提供了shell脚本处理文件的工具类
package com.lancy.common.util;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.HashMap;
import java.util.Map;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
public class ShellUtil {
private static Logger logger = LogManager.getLogger(ShellUtil.class);
/**
* 获取文件内某一列的总数,并且是以tab为分隔符
*
* @param file
* 文件路径
* @param colNum
* 列号 由1开始
* @return 总和
*/
public static double sumColumn(String file, int colNum) {
String cmds = "awk -F '\t' '{a=a+$" + colNum + "}END{printf(\"%.3f\\n\", a)}' " + file;
String result = execWithChannel(cmds);
return result == null || "".equals(result) ? 0 : Double.parseDouble(result);
}
/**
* 获取文件总行数 效率相对与直接调用java方法慢，不建议使用
*
* @param file
* 文件路径
* @return 总行数
*/
@Deprecated
public static int countLines(String file) {
String cmd = "awk 'END{print NR}' " + file;
String result = execWithChannel(cmd);
return result == null || "".equals(result) ? 0 : Integer.parseInt(result);
}
/**
* 执行MongoDB数据导入脚本
*
* @param filePath
* 文件路径
* @return
*/
public static String execMongoImportScript(String filePath, String collection, String field) {
String cmds = "source /etc/profile && sh ${PROG_ROOT}/shell/script/mongo_data_import.sh " + filePath + " "
+ collection + " " + field;
return execWithChannel(cmds);
}
/**
* 执行MongoDB数据导入脚本
*
* @param filePath
* 文件路径
* @return
*/
public static String execMongoImportScript(String filePath, String collection, String field, String cardType) {
if (cardType == null || cardType.length() <= 0) {
return execMongoImportScript(filePath, collection, field);
}
String cmds = "source /etc/profile && sh ${PROG_ROOT}/shell/script/mongo_data_import.sh " + filePath + " "
+ collection + " " + field + " " + cardType;
return execWithChannel(cmds);
}
/**
* 执行MongoDB票卡导出-去重-导入脚本
*
* @param fileDir
* 票卡数据输出临时目录
* @return
*/
public static String execMongoCardScript(String fileDir, String host, String db) {
String cmds = "sh /home/lnt/shell/script/mongo_card_import.sh " + fileDir + " " + host + " " + db;
return execWithChannel(cmds);
}
/**
* 执行shell命令
*
* @param cmds
* shell命令
* @return
*/
public static String execute(String cmds) {
return execWithChannel(cmds);
}
/**
* 文件数据重复性检查
*
* @param filePath
* 原始文件路径，字符串形式
* @param fields
* 查重字段，参数样式： new String[]{"1","2"}
* @return 结果文件，在原始文件每行记录前面添加一个标识，0表示未重复，1表示重复，原始记录与标识以&分隔
*/
public static File repetitionCheck(String filePath, String[] fields) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < fields.length; i++) {
sb.append("$").append(fields[i]);
}
File srcFile = new File(filePath);
File tmpFile = new File(filePath + ".TMP");
StringBuilder cmds = new StringBuilder();
cmds.append("awk '{if(record[").append(sb.toString()).append("]==0){record[").append(sb.toString())
.append("]++;flag=0;}else{flag=1;}print flag\"&\"$0}' ").append(srcFile).append(" > ").append(tmpFile);
String result = execWithChannel(cmds.toString());
System.out.println(result);
return tmpFile;
}
/**
* 文件内排序
*
* @param file
* 文件路径
* @param dist
* 保存到的文件路径，可为源文件
* @param column
* 排序的列
*/
public static void sort(String file, String dist, int column) {
sort(file, dist, new int[] { column }, false);
}
/**
* 文件内排序
*
* @param file
* 文件路径
* @param dist
* 保存到的文件路径，可为源文件
* @param column
* 排序的列
* @param desc
* 是否倒序
*/
public static void sort(String file, String dist, int column, boolean desc) {
sort(file, dist, new int[] { column }, desc);
}
/**
* 文件内排序
*
* @param file
* 文件路径
* @param dist
* 保存到的文件路径，可为源文件
* @param columns
* 排序的多列
* @param desc
* 是否倒序
*/
public static void sort(String file, String dist, int[] columns, boolean desc) {
StringBuilder cmds = new StringBuilder("sort");
for (int column : columns) {
cmds.append(" -k").append(column);
if (desc) {
cmds.append("r");
}
cmds.append(",").append(column);
}
cmds.append(" ").append(file);
if (dist != null && dist.length() > 0) {
if (dist.equals(file)) {
cmds.append(" -o ");
} else {
cmds.append(" > ");
}
cmds.append(dist);
}
System.out.println(cmds.toString());
String result = execWithChannel(cmds.toString());
System.out.println(result);
}
/**
* 文件内排序，支持不同列可具有不同的排序方式（正序或倒序）
*
* @param file
* 文件路径
* @param dist
* 保存到的文件路径，可为源文件
* @param columns
* 排序的多列
*/
public static void sort(String file, String dist, Map columns) {
StringBuilder cmds = new StringBuilder("sort");
for (Map.Entry entry : columns.entrySet()) {
cmds.append(" -k").append(entry.getKey());
if (entry.getValue()) {
cmds.append("r");
}
cmds.append(",").append(entry.getKey());
}
cmds.append(" ").append(file);
if (dist != null && dist.length() > 0) {
if (dist.equals(file)) {
cmds.append(" -o ");
} else {
cmds.append(" > ");
}
cmds.append(dist);
}
System.out.println(cmds.toString());
String result = execWithChannel(cmds.toString());
System.out.println(result);
}
/**
* 带管道符命令执行方法
*
* @param cmds
* @return
*/
private static String execWithChannel(String cmds) {
Process process = null;
BufferedInputStream in = null;
BufferedReader reader = null;
StringBuilder result = new StringBuilder();
try {
process = Runtime.getRuntime().exec(new String[] { "/bin/sh", "-c", cmds });
in = new BufferedInputStream(process.getInputStream());
reader = new BufferedReader(new InputStreamReader(in));
String lineStr = null;
while ((lineStr = reader.readLine()) != null) {
result.append(lineStr.trim()).append("\n");
}
int exitValue = process.waitFor();
logger.info(">>>shell执行命令返回结果:exitValue = " + exitValue);
} catch (IOException | InterruptedException e) {
logger.error(">>>shell执行命令异常>>>" + e.getMessage());
result.append("999999999999");
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException e) {
logger.error(">>>shell执行命令异常>>>" + e.getMessage());
}
}
if (in != null) {
try {
in.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
logger.info(">>>shell执行命令返回结果>>>" + result.toString());
return result.toString();
}
/**
* 执行命令
*
* @param cmds
* @return
*/
private static String exec(String[] cmds) {
Process process = null;
BufferedInputStream in = null;
BufferedReader reader = null;
StringBuilder result = new StringBuilder();
try {
process = Runtime.getRuntime().exec(cmds);
in = new BufferedInputStream(process.getInputStream());
reader = new BufferedReader(new InputStreamReader(in));
String lineStr = null;
while ((lineStr = reader.readLine()) != null) {
result.append(lineStr.trim()).append("\n");
}
} catch (IOException e) {
e.printStackTrace();
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (in != null) {
try {
in.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return result.toString();
}
}