Commit 46b3653a authored by wangjinjing's avatar wangjinjing

定时统计前一天的车流量及删除2个月前的车流量

parent 52ea66f1
The MIT License (MIT)
Copyright (c) 2018 杨东川
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
\ No newline at end of file
# Excel_To_DB
一款将Excel表格中的数据导入至数据库中的小工具,SpringBoot+EasyPoi+Redis消息队列实现Excel批量异步导入数据库
详细介绍:
1. [【Excel_To_DB】SpringBoot+EasyPoi+Redis消息队列实现Excel批量异步导入数据库(一)](http://blog.csdn.net/yangdongchuan1995/article/details/79277834)
2. [【Excel_To_DB】SpringBoot+EasyPoi+Redis消息队列实现Excel批量异步导入数据库(二)](http://blog.csdn.net/YangDongChuan1995/article/details/79285341)
3. [【Excel_To_DB】SpringBoot+EasyPoi+Redis消息队列实现Excel批量异步导入数据库(三)](http://blog.csdn.net/yangdongchuan1995/article/details/79290027)
This diff is collapsed.
package com.hzjt.config;
import com.hzjt.handler.WebSocket;
import com.hzjt.service.TraffFlowService;
import com.hzjt.service.TraffalarmrecordService;
import com.hzjt.util.DateUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
......@@ -10,83 +13,42 @@ import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.net.URI;
import java.util.Calendar;
/**
* 定时任务
*
* @author tycoding
* @date 2019-06-17
*/
//@Slf4j
//@Component
//@Configuration
//@EnableScheduling
@Slf4j
@Component
@Configuration
@EnableScheduling
public class ScheduleTaskConfig {
// private static final Long MINUTE_30 = 1000 * 60 * 30L;
//
//// @Value("${ip.host}")
//// private String iptr;
//
// @Scheduled(cron = "59 * * * * ?")//每59s 触发一次
// private void reconnectWebsocket() {
//
//// WebSocket webSocket=new WebSocket();
// //根据连接的name ,群发根据videoid 查询的结果
//
//
//
//// String[] ips = iptr.split(",");
//// for (String ip : ips) {
//// try {
////// MyWebSocketClient client = new MyWebSocketClient(new URI("ws://" + ip + "8001/api/message/sub/traffic-incident/alarm"));
////// if (!client.isOpen()) {
////// if (client.getReadyState().equals(WebSocket.READYSTATE.NOT_YET_CONNECTED)) {
////// try {
////// client.connect();
////// } catch (IllegalStateException e) {
////// }
////// } else if (client.getReadyState().equals(WebSocket.READYSTATE.CLOSING) || client.getReadyState().equals(WebSocket.READYSTATE.CLOSED)) {
////// client.reconnect();
////// }
////// }
////
////
//// }catch (Exception ex){
////
////
//// }
// } private static final Long MINUTE_30 = 1000 * 60 * 30L;
//
//// @Value("${ip.host}")
//// private String iptr;
//
// @Scheduled(cron = "59 * * * * ?")//每59s 触发一次
// private void reconnectWebsocket() {
//
//// WebSocket webSocket=new WebSocket();
// //根据连接的name ,群发根据videoid 查询的结果
//
//
//
//// String[] ips = iptr.split(",");
//// for (String ip : ips) {
//// try {
////// MyWebSocketClient client = new MyWebSocketClient(new URI("ws://" + ip + "8001/api/message/sub/traffic-incident/alarm"));
////// if (!client.isOpen()) {
////// if (client.getReadyState().equals(WebSocket.READYSTATE.NOT_YET_CONNECTED)) {
////// try {
////// client.connect();
////// } catch (IllegalStateException e) {
////// }
////// } else if (client.getReadyState().equals(WebSocket.READYSTATE.CLOSING) || client.getReadyState().equals(WebSocket.READYSTATE.CLOSED)) {
////// client.reconnect();
////// }
////// }
////
////
//// }catch (Exception ex){
////
////
//// }
// }
@Autowired
TraffFlowService traffFlowService;
@Autowired
TraffalarmrecordService traffalarmrecordService;
@Scheduled(cron = "0 0 2 * * ? ")//每天凌晨2点执行
private void statis() {
log.info("每天凌晨2点执行前一天数据整合");
String date = DateUtils.getYesterday(-1);
//抽取前一天的车流量和事件统计数据入表
Integer result = traffFlowService.statisVehiclesByDay(date);
//抽取前一天的事件统计到新表中
Integer resultrecord = traffalarmrecordService.statisTraffalarmrecordstatByDay(date);
}
@Scheduled(cron = "0 0 23 28-31 * ?")//每一个月执行一次
private void reconnectWebsocket() {
final Calendar c = Calendar.getInstance();
if (c.get(Calendar.DATE) == c.getActualMaximum(Calendar.DATE)) {
log.info("每个月最后一天执行删除前2个月车流量数据操作start");
//删除2个月之前车流量详情的数据
traffFlowService.deleteBeforeTwoMonthVehicles(DateUtils.getlastMonth(-2));
//删除2个月之前的车流量数据
traffFlowService.deleteBeforeTwoMonthVehicles(DateUtils.getlastMonth(-2));
log.info("每个月最后一天执行删除前2个月车流量数据操作end");
}
}
}
package com.hzjt.config;
import org.apache.tomcat.util.threads.ThreadPoolExecutor;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.TaskExecutor;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
@Configuration
@EnableAsync
public class ThreadPoolConfig {
@Bean
public TaskExecutor taskExecutor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
// 设置核心线程数
executor.setCorePoolSize(5);
// 设置最大线程数
executor.setMaxPoolSize(10);
// 设置队列容量
executor.setQueueCapacity(20);
// 设置线程活跃时间(秒)
executor.setKeepAliveSeconds(60);
// 设置默认线程名称
executor.setThreadNamePrefix("hello-");
// 设置拒绝策略
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
// 等待所有任务结束后再关闭线程池
executor.setWaitForTasksToCompleteOnShutdown(true);
return executor;
}
}
......@@ -21,21 +21,6 @@ public interface TraffAlarmRecordMapper extends BaseMapper<Traffalarmrecord>, Co
int selectmax();
List<TraffalarmrecordResult> queryTraffalarmrecordByPage(@Param("recordtype")String recordtype, @Param("areaid")Long areaid, @Param("checkstatus")Integer checkstatus,
@Param("starttime")String starttime, @Param("endtime")String endtime,
@Param("deviceChannelids")List<DeviceChannelid> deviceChannelid,
@Param("userAccount") String userAccount, @Param("pushstatus")Integer pushstatus,
@Param("page")Integer page, @Param("rows")Integer rows);
int countQueryTraffalarmrecordByPage(@Param("recordtype")String recordtype,@Param("areaid")Long areaid, @Param("checkstatus")Integer checkstatus,
@Param("starttime")String starttime, @Param("endtime")String endtime,
@Param("deviceChannelids")List<DeviceChannelid> deviceChannelid,
@Param("userAccount") String userAccount,@Param("pushstatus")Integer pushstatus,
@Param("page")Integer page,@Param("rows")Integer rows);
List<TraffalarmrecordResult> queryTraffalarmrecordAllByPage(@Param("recordtype")String recordtype,
@Param("areaid")Long areaid,
@Param("checkstatus")Integer checkstatus);
Integer statisTraffalarmrecordstatByDay(String starttime);
}
\ No newline at end of file
......@@ -16,6 +16,8 @@ public interface TraffFlowMapper {
List<Map> selectFiveAndDayFlow(String videoid);
List<Map> selectFiveAndTypeDayFlow(String videoid);
Integer statisVehiclesByDay(String startime);
Integer deleteBeforeTwoMonthVehicles(String startime);
Integer deleteBeforeTwoMonthVehiclesDetails(String startime);
}
package com.hzjt.redis;
import com.hzjt.domain.Alarm;
import com.hzjt.domain.Ftp;
import com.hzjt.domain.Traffalarmrecord;
import com.hzjt.handler.DelayQueueManager;
import com.hzjt.handler.FileTransferManager;
import com.hzjt.mapper.TraffAlarmRecordMapper;
import com.hzjt.service.AlarmRecordStatisticsService;
import com.hzjt.service.FtpService;
import com.hzjt.service.ImportService;
import com.hzjt.service.SeqService;
import com.hzjt.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* @Description: 消息接收者,将其在ExcelToDbApplication.java中注入消息监听容器(MessageListenerAdapter)中
* @Author: 杨东川【http://blog.csdn.net/yangdongchuan1995】
* @Date: Created in 2018-2-6
*/
@Service
public class Receiverbak {
@Autowired
ImportService importService;
@Autowired
RedisDao redisDao;
@Autowired
private TraffAlarmRecordMapper traffAlarmRecordMapper;
@Autowired
FtpService ftpService;
@Autowired
AlarmRecordStatisticsService alarmRecordStatisticsService;
@Value("ftpIp")
String ftpIp;
@Value("ftpPort")
private String ftpPort;
@Value("ftpUsername")
private String ftpUsername;
@Value("ftpPassword")
private String ftpPassword;
@Value("${alarmrecord.check.enable}")
private String checkEnable;
@Autowired
private SeqService seqService;
private Long delayTime = 25L;
// @Autowired
// FtpUtil ftpUtil;
@Autowired
private DelayQueueManager delayQueueManager;
private static final Logger log = LoggerFactory.getLogger(Receiverbak.class);
/**
* @Description: 用于接收单个对象,将对象同步至数据库,如果同步失败,则存入redis中
* @Param: [message] “fastjson”转换后的json字符串
* @Retrun: void
*/
public void receiveSinglewww(String Message) throws InterruptedException {
// 将json字符串转换成实体对
Alarm trffClientMessage= JsonUtil.stringToBean(Message,Alarm.class);
List<String> imgBase64List = trffClientMessage.getImg_base64();
Traffalarmrecord traffAlarmRecord = new Traffalarmrecord();
String imgEnumHead = "IMG";
//Ftp ftp = new Ftp(ftpIp,Integer.parseInt(ftpPort),ftpUsername,ftpPassword);
Ftp ftp = ftpService.reloadFtp();
for (int i = 0; i < imgBase64List.size(); i++) {
/* 图片上传 */
String path = trffClientMessage.getVideo_id() + "/" + DateUtils.formatCurrDayNoSign();
String fileName = UUIDUtils.createuuid() + ".jpg";
if (FTPUtil.uploadFile(ftp, path, fileName, imgBase64List.get(i))) {
TraffAlarmRecordImgEnum.valueOf(imgEnumHead + i).setImg(traffAlarmRecord, FTPUtil.getFtpUrl(ftp) + path + "/" + fileName);
}
}
List<String> imgUrls = trffClientMessage.getImg_urls();
for (int i = 0; i < imgUrls.size(); i++) {
TraffAlarmRecordFromImgEnum.valueOf(imgEnumHead + i).setImg(traffAlarmRecord, imgUrls.get(i));
}
String[] videoIdArr = trffClientMessage.getVideo_id().split("_");
String fdid = videoIdArr[0];
traffAlarmRecord.setFdid(fdid);
Integer channelid = Integer.valueOf(videoIdArr[1]) + 1;
traffAlarmRecord.setChannelid(channelid);
if (StringEnum.ONE.getValue().equals(checkEnable)) {
/* 9:免审 */
traffAlarmRecord.setCheckstatus(9);
}
int recordid = traffAlarmRecordMapper.selectmax();
traffAlarmRecord.setRecordid((long)recordid);
String dept = trffClientMessage.getDept();
traffAlarmRecord.setAreaid(Long.valueOf(dept));
String recordtype = trffClientMessage.getIncident_type().toLowerCase();
traffAlarmRecord.setRecordtype(recordtype);
traffAlarmRecord.setVideourlfrom(trffClientMessage.getVideo_record_url());
traffAlarmRecord.setRecordtime(new Date(Long.valueOf(trffClientMessage.getTs())));
traffAlarmRecord.setCreatetime(new Date());
traffAlarmRecordMapper.inserTraffAlarmRecord(traffAlarmRecord);
try {
System.out.println("fetch ftp:" + ftp.toString()+"recordId:"+traffAlarmRecord.getRecordid());
String basepath = "gstraff/" + traffAlarmRecord.getFdid() + (traffAlarmRecord.getChannelid() < 10 ? "0" + traffAlarmRecord.getChannelid() : traffAlarmRecord.getChannelid()) + "/" + DateUtils.formatCurrDayNoSign();
Map<String, Object> transferRecordMap = new HashMap<>();
FileTransferManager.fetchUrlsFromRecord(traffAlarmRecord, transferRecordMap);
FileTransferManager.transferFile(transferRecordMap, ftp, basepath);
// Traffalarmrecord recordBak = FileTransferManager.traffAlarmRecordUrlUpdate(results);
// traffAlarmRecord.setImg1path(recordBak.getImg1path());
// traffAlarmRecord.setImg2path(recordBak.getImg2path());
// traffAlarmRecord.setImg3path(recordBak.getImg3path());
// traffAlarmRecord.setImg4path(recordBak.getImg4path());
// traffAlarmRecord.setImg5path(recordBak.getImg5path());
// traffAlarmRecord.setVideopath(recordBak.getVideopath());
if (traffAlarmRecord != null && traffAlarmRecord.getRecordid() != null) {
traffAlarmRecordMapper.updateTraffAlarmRecordUrl(traffAlarmRecord);
}
} catch (Exception e) {
log.error("解析事件告警数据异常 :" + e.getMessage());
}
//
redisDao.incrOrDecr(Constant.succSizeTempKey, -1);
}
}
......@@ -5,6 +5,7 @@ import com.hzjt.domain.Vehiclesdetail;
import com.hzjt.mapper.TraffFlowMapper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
import java.util.Map;
......@@ -30,5 +31,14 @@ public class TraffFlowService {
return traffFlowMapper.selectFiveAndTypeDayFlow(videoid);
}
public Integer statisVehiclesByDay(String starttime) {
return traffFlowMapper.statisVehiclesByDay(starttime);
}
@Transactional(rollbackFor = Exception.class)
public void deleteBeforeTwoMonthVehicles(String starttime) {
traffFlowMapper.deleteBeforeTwoMonthVehiclesDetails(starttime);
traffFlowMapper.deleteBeforeTwoMonthVehicles(starttime);
}
}
\ No newline at end of file
......@@ -15,36 +15,9 @@ public class TraffalarmrecordService {
@Autowired
private TraffAlarmRecordMapper traffalarmrecordMapper;
private String fileagentUrl;
@Autowired
public TraffalarmrecordService(TraffAlarmRecordMapper traffalarmrecordMapper,
@Value("${gs.traff.cloud.web.fileagent.url}") String fileagentUrl) {
this.traffalarmrecordMapper = traffalarmrecordMapper;
this.fileagentUrl = fileagentUrl;
public Integer statisTraffalarmrecordstatByDay(String starttime) {
return traffalarmrecordMapper.statisTraffalarmrecordstatByDay(starttime);
}
public void saveTraffalarmrecord(Traffalarmrecord traffalarmrecord) {
traffalarmrecordMapper.insertSelective(traffalarmrecord);
}
public void saveTraffalarmrecordList(List<Traffalarmrecord> traffalarmrecordList) {
traffalarmrecordMapper.insertList(traffalarmrecordList);
}
public void deleteTraffalarmrecord(Traffalarmrecord traffalarmrecord) {
traffalarmrecordMapper.deleteByPrimaryKey(traffalarmrecord);
}
public void deleteTraffalarmrecordList(List<Traffalarmrecord> traffalarmrecordList) {
for (Traffalarmrecord traffalarmrecord:traffalarmrecordList) {
traffalarmrecordMapper.deleteByPrimaryKey(traffalarmrecord);
}
}
public void updateTraffalarmrecord(Traffalarmrecord traffalarmrecord) {
traffalarmrecordMapper.updateByPrimaryKeySelective(traffalarmrecord);
}
public Traffalarmrecord queryTraffalarmrecord(Traffalarmrecord traffalarmrecord) {
return traffalarmrecordMapper.selectOne(traffalarmrecord);
}
public int countTraffalarmrecord(Traffalarmrecord traffalarmrecord) {
return traffalarmrecordMapper.selectCount(traffalarmrecord);
}
}
\ No newline at end of file
package com.hzjt.util;
/**
* @Description: 常量
* @Author: 杨东川【http://blog.csdn.net/yangdongchuan1995】
* @Date: Created in 2018-2-6
*/
public class Constant {
// 未通过格式校验的数据
public static final String failListKey = "excelToDB:failListKey";
......
......@@ -16,7 +16,7 @@ public class DateUtils {
private static final DateTimeFormatter YMD_HMS = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss");
private static final DateTimeFormatter YMDHMS = DateTimeFormat.forPattern("yyyyMMddHHmmss");
private static final DateTimeFormatter YMD = DateTimeFormat.forPattern("yyyyMMdd");
private static final DateTimeFormatter Y_M_D = DateTimeFormat.forPattern("yyyy-MM-dd");
public static String formatCurrDate(){
return formatCurrDateByType(YMD_HMS);
}
......@@ -37,6 +37,16 @@ public class DateUtils {
return DateTime.parse(date,YMDHMS).toDate();
}
public static String getYesterday(int day){
Date date= addDay(new Date(),day);
return new DateTime(date).toString(Y_M_D);
}
public static String getlastMonth(int month){
Date date= addMonth(new Date(),month);
return new DateTime(date).toString(Y_M_D);
}
public static String formatDate(Date date){
return new DateTime(date).toString(YMD_HMS);
}
......
......@@ -4,11 +4,6 @@ import com.alibaba.fastjson.JSON;
import java.util.List;
/**
* @Description: 工具类
* @Author: 杨东川【http://blog.csdn.net/yangdongchuan1995】
* @Date: Created in 2018-2-6
*/
public class JsonUtil {
public static <T> String beanToString(T value) {
if (value == null) {
......
......@@ -30,7 +30,7 @@ mybatis.configuration.default-statement-timeout=3000
# Mysql���ݿ�-����Դ����
#spring.datasource.username=test
#spring.datasource.password=test
#spring.datasource.url=jdbc:oracle:thin:@localhost:1521:orcl
#spring.datasource.url=jdbc:oracle:thin:@192.168.168.212:1521:helowin
spring.datasource.username=hzjt
spring.datasource.password=hzjt
spring.datasource.url=jdbc:oracle:thin:@33.50.1.22:1521:orcl
......
......@@ -4,7 +4,7 @@
<insert id="insertlist" parameterType="com.hzjt.domain.Vehiclesdetail" >
INSERT INTO vehiclesdetail (obj_id, vehiclesid, type, direction,classification_confidence,ruletag )
INSERT INTO vehiclesdetail (obj_id, vehiclesid, type, direction,classification_confidence,ruletag)
values( #{obj_id,jdbcType=VARCHAR},
#{vehiclesid,jdbcType=VARCHAR},
#{type,jdbcType=VARCHAR},
......@@ -83,4 +83,32 @@
</select>
<insert id="statisVehiclesByDay" >
insert into vehiclestatistic
select b.video_id ,
to_char(to_date(ts, 'yyyy-mm-dd hh24:mi:ss'), 'yyyy-mm-dd') ts,
t.type,
ruletag,
count(*) total
from VEHICLESDETAIL t, vehicles b
where t.vehiclesid = b.id
and ts>=#{starttime}|| ' 00:00:00'
and ts<![CDATA[ <= ]]>#{starttime}||' 23:59:59'
group by to_char(to_date(ts, 'yyyy-mm-dd hh24:mi:ss'), 'yyyy-mm-dd'),
t.type,
ruletag,
b.video_id
</insert>
<delete id="deleteBeforeTwoMonthVehiclesDetails">
delete from vehiclesdetail WHERE EXISTS
(
SELECT 1
FROM vehicles t
WHERE t.id = vehiclesid and ts<![CDATA[ <= ]]>#{starttime}||' 23:59:59'
)
</delete>
<delete id="deleteBeforeTwoMonthVehicles">
delete from vehicles where ts<![CDATA[ <= ]]>#{starttime}||' 23:59:59'
</delete>
</mapper>
\ No newline at end of file
......@@ -112,4 +112,21 @@
</select>
<insert id="statisTraffalarmrecordstatByDay" >
insert into TRAFFALARMRECORDSTATE
SELECT A.FDID,CHANNELID, to_char(A.RECORDTIME,'yyyy-MM-DD')countdate,count(*) totalcount,recordtype eventtype,areaid,
(select B.TDMC from SBTDSPSR B where A.FDID=B.SBBH and CHANNELID=B.TDBH and rownum=1)tdmc,
(select C.NAME from T_CODE C where C.KEY=UPPER(RECORDTYPE)and rownum=1 ) eventtypename,
(SELECT DEPT_NAME FROM Dept d where d.dept_id=a.areaid and rownum=1)areaname,
sum( case when pushstatus is null or pushstatus=9 then 1 else 0 end)nopush,
sum( case when pushstatus =0 then 1 else 0 end)successpush,
sum( case when pushstatus =1 then 1 else 0 end)failpush
FROM TRAFFALARMRECORD A
where A.RECORDTIME>=to_date(#{starttime}||' 00:00:00','YYYY-MM-DD HH24:MI:SS')
and A.RECORDTIME<![CDATA[ <= ]]>to_date(#{starttime}||' 23:59:59','YYYY-MM-DD HH24:MI:SS')
group by to_char(A.RECORDTIME,'yyyy-MM-DD'),recordtype,A.FDID,CHANNELID,areaid
</insert>
</mapper>
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment