feat(weather): 添加历史天气数据同步逻辑

- 在定时任务中增加首次同步判断,若没有历史数据则从最早的气象站记录开始同步
- 新增获取最早气象站数据时间的方法,用于确定历史同步起点
- 添加按地区编码和时间范围批量查询气象数据的方法,优化数据获取性能
- 更新.gitignore文件,排除CLAUDE.md文件
This commit is contained in:
fsyud 2026-04-14 15:33:14 +08:00
parent ee27a7130b
commit 2f65f85143
8 changed files with 75 additions and 5 deletions

1
.gitignore vendored
View File

@ -8,6 +8,7 @@
**/**/target/ **/**/target/
.fastRequest .fastRequest
.qoder .qoder
CLAUDE.md
### IntelliJ IDEA ### ### IntelliJ IDEA ###
.idea .idea
*.iws *.iws

View File

@ -42,4 +42,16 @@ public interface RegionalWeatherDataMapper extends BaseMapper<RegionalWeatherDat
* @return 实体列表 * @return 实体列表
*/ */
List<RegionalWeatherData> selectByCondition(Map<String, Object> params); List<RegionalWeatherData> selectByCondition(Map<String, Object> params);
/**
* 批量查询气象数据根据多个地区编码和时间范围
* @param orgCodes 地区编码列表
* @param startTimeData 起始时间整时格式 2026-04-13 10:00:00
* @param endTimeData 结束时间整时格式 2026-04-13 15:00:00
* @return 实体列表
*/
List<RegionalWeatherData> selectByOrgCodesAndTimeRange(
@Param("orgCodes") List<String> orgCodes,
@Param("startTimeData") String startTimeData,
@Param("endTimeData") String endTimeData);
} }

View File

@ -22,4 +22,6 @@ public interface RegionalWeatherStationMapper extends BaseMapper<RegionalWeather
); );
List<RegionalWeatherStation> getSyncData(@Param("prevHourDate") String prevHourDate); List<RegionalWeatherStation> getSyncData(@Param("prevHourDate") String prevHourDate);
String getSyncOldDataStartDate();
} }

View File

@ -16,4 +16,12 @@ public interface IRegionalWeatherDataService {
* @param endDate 结束日期(yyyy-mm-dd hh:mm)整时时间 * @param endDate 结束日期(yyyy-mm-dd hh:mm)整时时间
*/ */
void syncOldWeatherData(String startDate, String endDate); void syncOldWeatherData(String startDate, String endDate);
/**
* 获取同步旧开始日期
*
* @return 字符串
*/
String getSyncOldStartDate();
} }

View File

@ -3,17 +3,20 @@ package com.southern.power.grid.service.impl;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.southern.power.grid.config.BusinessThreadPoolFactory; import com.southern.power.grid.config.BusinessThreadPoolFactory;
import com.southern.power.grid.dao.RegionalWeatherDataMapper; import com.southern.power.grid.dao.RegionalWeatherDataMapper;
import com.southern.power.grid.dao.RegionalWeatherStationMapper;
import com.southern.power.grid.entity.RegionalWeatherData; import com.southern.power.grid.entity.RegionalWeatherData;
import com.southern.power.grid.service.IRegionalWeatherDataService; import com.southern.power.grid.service.IRegionalWeatherDataService;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Objects;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.Semaphore; import java.util.concurrent.Semaphore;
@ -24,7 +27,7 @@ import java.util.concurrent.Semaphore;
public class RegionalWeatherDataServiceImpl extends ServiceImpl<RegionalWeatherDataMapper, RegionalWeatherData> public class RegionalWeatherDataServiceImpl extends ServiceImpl<RegionalWeatherDataMapper, RegionalWeatherData>
implements IRegionalWeatherDataService { implements IRegionalWeatherDataService {
private final RegionalWeatherStationMapper regionalWeatherStationMapper;
private final BusinessThreadPoolFactory businessThreadPoolFactory; private final BusinessThreadPoolFactory businessThreadPoolFactory;
private final RegionalWeatherDataSyncService regionalWeatherDataSyncService; private final RegionalWeatherDataSyncService regionalWeatherDataSyncService;
@ -123,5 +126,18 @@ public class RegionalWeatherDataServiceImpl extends ServiceImpl<RegionalWeatherD
} }
} }
@Override
public String getSyncOldStartDate() {
Long count = this.lambdaQuery().count();
if (!Objects.isNull(count) && count > 0) {
return null;
}
String startDate = regionalWeatherStationMapper.getSyncOldDataStartDate();
if (StringUtils.hasLength(startDate)) {
return startDate;
}
return null;
}
} }

View File

@ -4,6 +4,7 @@ import com.southern.power.grid.service.IRegionalWeatherDataService;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.time.LocalDateTime; import java.time.LocalDateTime;
@ -34,14 +35,24 @@ public class WeatherDataScheduleTask {
*/ */
private void execute() { private void execute() {
if (!running.compareAndSet(false, true)) { if (!running.compareAndSet(false, true)) {
log.warn("任务正在执行,跳过本次触发"); log.warn("【区域天气数据同步任务】任务正在执行,跳过本次触发");
return; return;
} }
try { try {
//第一次同步 获取历史数据的开始日期
String startDate = regionalWeatherDataService.getSyncOldStartDate();
if (StringUtils.hasLength(startDate)) {
log.info("【区域天气数据同步任务】不存在同步数据,开始同步历史数据,历史数据开始日期:{}", startDate);
//同步所有的历史数据
regionalWeatherDataService.syncOldWeatherData(startDate, LocalDateTime.now().minusHours(1).format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:00:00")));
} else {
//获取当前时间前一个整点时间 //获取当前时间前一个整点时间
String prevHourDate = LocalDateTime.now().minusHours(1).format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:00:00")); String prevHourDate = LocalDateTime.now().minusHours(1).format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:00:00"));
log.info("【区域天气数据同步任务】:存在同步数据,开始同步上一小时数据:{}", prevHourDate);
regionalWeatherDataService.scheduleSyncWeatherData(prevHourDate); regionalWeatherDataService.scheduleSyncWeatherData(prevHourDate);
}
log.info("【区域天气数据同步任务】定时触发--结束"); log.info("【区域天气数据同步任务】定时触发--结束");
} catch (Exception e) { } catch (Exception e) {
log.error("【区域天气数据同步任务】执行异常", e); log.error("【区域天气数据同步任务】执行异常", e);

View File

@ -82,4 +82,17 @@
</where> </where>
ORDER BY create_time DESC ORDER BY create_time DESC
</select> </select>
<!-- 批量查询气象数据:根据多个地区编码和时间范围,只查映射所需字段 -->
<select id="selectByOrgCodesAndTimeRange" resultMap="BaseResultMap">
SELECT org_code, data_time, hourly_precipitation, daily_precipitation,
temperature, hourly_max_temperature, hourly_min_temperature, extreme_wind_speed_hourly
FROM regional_weather_data
WHERE org_code IN
<foreach collection="orgCodes" item="code" open="(" separator="," close=")">
#{code}
</foreach>
AND data_time BETWEEN #{startTimeData, jdbcType=VARCHAR} AND #{endTimeData, jdbcType=VARCHAR}
ORDER BY org_code, data_time ASC
</select>
</mapper> </mapper>

View File

@ -92,4 +92,11 @@
from regional_weather_station from regional_weather_station
where data_time = #{prevHourDate} where data_time = #{prevHourDate}
</select> </select>
<select id="getSyncOldDataStartDate" resultType="java.lang.String">
SELECT data_time
FROM regional_weather_station
ORDER BY data_time
LIMIT 1
</select>
</mapper> </mapper>