commit 1632d5efbc702cfdb91547d24f9bc41a104af9e7 parent 935e322f1b0278147bd6e4ae26ea99edc7b3678a Author: Kebigon <git@kebigon.xyz> Date: Thu, 5 Mar 2020 21:09:56 +0900 Work in progress Diffstat:
| A | .gitignore | | | 116 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | pom.xml | | | 93 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/assembly/assembly.xml | | | 34 | ++++++++++++++++++++++++++++++++++ |
| A | src/main/java/xyz/kebigon/housesearch/ApplicationContext.java | | | 18 | ++++++++++++++++++ |
| A | src/main/java/xyz/kebigon/housesearch/HouseSearchApplication.java | | | 72 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/main/java/xyz/kebigon/housesearch/browser/Browser.java | | | 54 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/main/java/xyz/kebigon/housesearch/browser/suumo/SuumoBrowser.java | | | 86 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/main/java/xyz/kebigon/housesearch/browser/suumo/SuumoSearchURLBuilder.java | | | 260 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/main/java/xyz/kebigon/housesearch/browser/yahoo/transit/YahooTransitBrowser.java | | | 121 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/main/java/xyz/kebigon/housesearch/browser/yahoo/transit/YahooTransitSearchURLBuilder.java | | | 35 | +++++++++++++++++++++++++++++++++++ |
| A | src/main/java/xyz/kebigon/housesearch/domain/AnounceType.java | | | 18 | ++++++++++++++++++ |
| A | src/main/java/xyz/kebigon/housesearch/domain/Area.java | | | 18 | ++++++++++++++++++ |
| A | src/main/java/xyz/kebigon/housesearch/domain/Posting.java | | | 67 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/main/java/xyz/kebigon/housesearch/domain/Route.java | | | 97 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/main/java/xyz/kebigon/housesearch/domain/SearchConditions.java | | | 96 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/main/java/xyz/kebigon/housesearch/domain/SearchConditionsValidator.java | | | 149 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/main/java/xyz/kebigon/housesearch/file/RoutesCache.java | | | 55 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/main/java/xyz/kebigon/housesearch/file/SearchArchive.java | | | 59 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/main/java/xyz/kebigon/housesearch/mail/EmailSender.java | | | 132 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | src/main/packaged-resources/cfg/log4j2.xml | | | 20 | ++++++++++++++++++++ |
| A | src/main/packaged-resources/cfg/mail.cfg | | | 11 | +++++++++++ |
| A | src/main/packaged-resources/cfg/search-conditions.cfg | | | 22 | ++++++++++++++++++++++ |
| A | src/main/packaged-resources/cfg/search-expression.cfg | | | 25 | +++++++++++++++++++++++++ |
| A | src/test/java/xyz/kebigon/suumo/browser/SearchURLBuilderTest.java | | | 47 | +++++++++++++++++++++++++++++++++++++++++++++++ |
24 files changed, 1705 insertions(+), 0 deletions(-)
diff --git a/.gitignore b/.gitignore @@ -0,0 +1,116 @@ +# +# Eclipse +# + +.metadata +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.settings/ +.loadpath +.recommenders + +# Eclipse Core +.project + +# External tool builders +.externalToolBuilders/ + +# Locally stored "Eclipse launch configurations" +*.launch + +# PyDev specific (Python IDE for Eclipse) +*.pydevproject + +# CDT-specific (C/C++ Development Tooling) +.cproject + +# CDT- autotools +.autotools + +# JDT-specific (Eclipse Java Development Tools) +.classpath + +# Java annotation processor (APT) +.factorypath + +# PDT-specific (PHP Development Tools) +.buildpath + +# sbteclipse plugin +.target + +# Tern plugin +.tern-project + +# TeXlipse plugin +.texlipse + +# STS (Spring Tool Suite) +.springBeans + +# Code Recommenders +.recommenders/ + +# Annotation Processing +.apt_generated/ +.apt_generated_test/ + +# Scala IDE specific (Scala & Java development for Eclipse) +.cache-main +.scala_dependencies +.worksheet + +# Uncomment this line if you wish to ignore the project description file. +# Typically, this file would be tracked if it contains build/dependency configurations: +#.project + + +# +# Java +# + +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.nar +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + + +# +# Maven +# + +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties +# https://github.com/takari/maven-wrapper#usage-without-binary-jar +.mvn/wrapper/maven-wrapper.jar diff --git a/pom.xml b/pom.xml @@ -0,0 +1,93 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <groupId>xyz.kebigon</groupId> + <artifactId>suumo-search</artifactId> + <version>0.0.1-SNAPSHOT</version> + + <name>suumo-search</name> + <description>Perform advanced searches on Suumo.jp</description> + + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <maven.compiler.source>1.8</maven.compiler.source> + <maven.compiler.target>1.8</maven.compiler.target> + </properties> + + <dependencies> + + <!-- Jackson Databind --> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>2.10.3</version> + </dependency> + + <!-- Apache Commons Email --> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-email</artifactId> + <version>1.5</version> + </dependency> + + <!-- Apache Log4j SLF4J Binding --> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-slf4j-impl</artifactId> + <version>2.13.1</version> + </dependency> + + <!-- Project Lombok --> + <dependency> + <groupId>org.projectlombok</groupId> + <artifactId>lombok</artifactId> + <version>1.18.12</version> + <scope>provided</scope> + </dependency> + + <!-- HtmlUnit Driver --> + <dependency> + <groupId>org.seleniumhq.selenium</groupId> + <artifactId>htmlunit-driver</artifactId> + <version>2.36.0</version> + </dependency> + + <!-- Spring expression --> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-expression</artifactId> + <version>5.2.4.RELEASE</version> + </dependency> + + <!-- JUnit Jupiter API --> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + <version>5.6.0</version> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <descriptors> + <descriptor>src/assembly/assembly.xml</descriptor> + </descriptors> + </configuration> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + +</project> diff --git a/src/assembly/assembly.xml b/src/assembly/assembly.xml @@ -0,0 +1,33 @@ +<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.0.0 http://maven.apache.org/xsd/assembly-2.0.0.xsd"> + <id>release</id> + <formats> + <format>tar.gz</format> + </formats> + <fileSets> + <fileSet> + <directory>src/main/packaged-resources</directory> + <outputDirectory>.</outputDirectory> + </fileSet> + <fileSet> + <directory>.</directory> + <outputDirectory>var/log</outputDirectory> + <excludes> + <exclude>*/**</exclude> + </excludes> + </fileSet> + </fileSets> + <dependencySets> + <dependencySet> + <outputDirectory>ext</outputDirectory> + <excludes> + <exclude>xyz.kebigon:suumo-search</exclude> + </excludes> + </dependencySet> + <dependencySet> + <outputDirectory>lib</outputDirectory> + <includes> + <include>xyz.kebigon:suumo-search</include> + </includes> + </dependencySet> + </dependencySets> +</assembly> +\ No newline at end of file diff --git a/src/main/java/xyz/kebigon/housesearch/ApplicationContext.java b/src/main/java/xyz/kebigon/housesearch/ApplicationContext.java @@ -0,0 +1,18 @@ +package xyz.kebigon.housesearch; + +import xyz.kebigon.housesearch.browser.yahoo.transit.YahooTransitBrowser; + +public class ApplicationContext +{ + private static YahooTransitBrowser yahooTransitBrowser; + + public static void setYahooTransitBrowser(YahooTransitBrowser yahooTransitBrowser) + { + ApplicationContext.yahooTransitBrowser = yahooTransitBrowser; + } + + public static YahooTransitBrowser getYahooTransitBrowser() + { + return yahooTransitBrowser; + } +} diff --git a/src/main/java/xyz/kebigon/housesearch/HouseSearchApplication.java b/src/main/java/xyz/kebigon/housesearch/HouseSearchApplication.java @@ -0,0 +1,72 @@ +package xyz.kebigon.housesearch; + +import java.io.IOException; +import java.util.Collection; +import java.util.stream.Collectors; + +import org.apache.commons.mail.EmailException; +import org.springframework.util.StringUtils; + +import lombok.extern.slf4j.Slf4j; +import xyz.kebigon.housesearch.browser.suumo.SuumoBrowser; +import xyz.kebigon.housesearch.browser.yahoo.transit.YahooTransitBrowser; +import xyz.kebigon.housesearch.domain.Posting; +import xyz.kebigon.housesearch.domain.SearchConditions; +import xyz.kebigon.housesearch.domain.SearchConditionsValidator; +import xyz.kebigon.housesearch.file.SearchArchive; +import xyz.kebigon.housesearch.mail.EmailSender; + +@Slf4j +public class HouseSearchApplication +{ + public static void main(String[] args) throws IOException, EmailException + { + final SearchConditions conditions = SearchConditions.load(); + + Collection<Posting> postings; + + try (final SearchArchive archive = new SearchArchive()) + { + try (final SuumoBrowser suumo = new SuumoBrowser()) + { + postings = suumo.search(conditions, archive); + } + } + + if (postings.isEmpty()) + { + log.info("No postings found on Suumo, terminating"); + return; + } + + if (!StringUtils.isEmpty(conditions.getExpression())) + { + try (final YahooTransitBrowser yahooTransit = new YahooTransitBrowser()) + { + ApplicationContext.setYahooTransitBrowser(yahooTransit); + + postings = postings.stream() // Do not parallel here + .filter(property -> SearchConditionsValidator.validateExpression(property, conditions)).collect(Collectors.toList()); + } + + if (postings.isEmpty()) + { + log.info("No postings left after applying expression filter, terminating"); + return; + } + } + + log.info("=======[ RESULTS ]======="); + log.info("Found {} postings", postings.size()); + + for (final Posting posting : postings) + log.info("-> {}", posting); + + log.info("Sending email notification"); + + final EmailSender sender = new EmailSender(); + sender.send(postings); + + log.info("Email notification sent, terminating"); + } +} diff --git a/src/main/java/xyz/kebigon/housesearch/browser/Browser.java b/src/main/java/xyz/kebigon/housesearch/browser/Browser.java @@ -0,0 +1,54 @@ +package xyz.kebigon.housesearch.browser; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; + +import org.openqa.selenium.By; +import org.openqa.selenium.NoSuchElementException; +import org.openqa.selenium.WebDriver; +import org.openqa.selenium.WebElement; +import org.openqa.selenium.htmlunit.HtmlUnitDriver; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class Browser implements Closeable +{ + private final WebDriver driver; + + protected Browser() + { + driver = new HtmlUnitDriver(); + } + + @Override + public void close() throws IOException + { + driver.quit(); + } + + protected void navigateTo(String url) + { + log.info("Navigate to {}", url); + driver.navigate().to(url); + } + + protected List<WebElement> findElements(String xpathExpression) + { + return driver.findElements(By.xpath(xpathExpression)); + } + + protected boolean click(String xpathExpression) + { + try + { + driver.findElement(By.xpath(xpathExpression)).click(); + return true; + } + catch (final NoSuchElementException e) + { + return false; + } + } +} diff --git a/src/main/java/xyz/kebigon/housesearch/browser/suumo/SuumoBrowser.java b/src/main/java/xyz/kebigon/housesearch/browser/suumo/SuumoBrowser.java @@ -0,0 +1,86 @@ +package xyz.kebigon.housesearch.browser.suumo; + +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Collection; +import java.util.stream.Collectors; + +import org.openqa.selenium.By; +import org.openqa.selenium.WebElement; + +import xyz.kebigon.housesearch.browser.Browser; +import xyz.kebigon.housesearch.domain.Posting; +import xyz.kebigon.housesearch.domain.SearchConditions; +import xyz.kebigon.housesearch.domain.SearchConditionsValidator; +import xyz.kebigon.housesearch.file.SearchArchive; + +public class SuumoBrowser extends Browser +{ + public Collection<Posting> search(SearchConditions conditions, SearchArchive archive) + { + navigateTo(SuumoSearchURLBuilder.build(conditions)); + + final Collection<Posting> postings = new ArrayList<Posting>(); + + do + { + postings.addAll(findElements("//div[@class='property_unit-content']").parallelStream() // + .map(SuumoBrowser::createPosting)// + .filter(archive::filter) // + .filter(posting -> SearchConditionsValidator.validateBasicConditions(posting, conditions)) // + .collect(Collectors.toList())); + } while (click("//a[text()='次へ']")); + + return postings; + } + + private static Posting createPosting(WebElement posting) + { + final String url = posting.findElement(By.xpath("./div[@class='property_unit-header']/h2/a")).getAttribute("href"); + + final String priceField = getField(posting, "販売価格"); + final int priceSubstringIndex = priceField.indexOf("万円"); + final long price = Long.parseLong(priceField.substring(0, priceSubstringIndex)) * 10000; + + final String ageField = getField(posting, "築年月"); + final int ageSubstringIndex = ageField.indexOf("年"); + final int age = LocalDate.now().getYear() - Integer.parseInt(ageField.substring(0, ageSubstringIndex)); + + final Double landSurface = parseSurfaceField(posting, "土地面積"); + final Double houseSurface = parseSurfaceField(posting, "建物面積"); + + // ＪＲ常磐線「荒川沖」徒歩33分 + final String stationField = getField(posting, "沿線・駅"); + + final int walkTimeToStationSubstringIndex = stationField.indexOf("徒歩"); + final Integer walkTimeToStation = walkTimeToStationSubstringIndex != -1 + ? Integer.parseInt(stationField.substring(walkTimeToStationSubstringIndex + 2, stationField.indexOf("分", walkTimeToStationSubstringIndex))) + : null; + + final int stationOpenBracketIndex = stationField.indexOf('「'); + final int stationCloseBracketIndex = stationField.indexOf('」'); + final String station = stationOpenBracketIndex != -1 && stationCloseBracketIndex != -1 + ? stationField.substring(stationOpenBracketIndex + 1, stationCloseBracketIndex) + : null; + + return new Posting(url, price, age, landSurface, houseSurface, walkTimeToStation, station); + } + + private static Double parseSurfaceField(WebElement posting, String fieldName) + { + final String surfaceField = getField(posting, fieldName); + + int surfaceSubstringIndex = surfaceField.indexOf("m2"); + if (surfaceSubstringIndex == -1) + surfaceSubstringIndex = surfaceField.indexOf("㎡"); + + return surfaceSubstringIndex != -1 ? // + Double.parseDouble(surfaceField.substring(0, surfaceSubstringIndex)) : // + null; + } + + private static String getField(WebElement posting, String fieldName) + { + return posting.findElement(By.xpath(".//dl[dt='" + fieldName + "']/dd")).getText(); + } +} diff --git a/src/main/java/xyz/kebigon/housesearch/browser/suumo/SuumoSearchURLBuilder.java b/src/main/java/xyz/kebigon/housesearch/browser/suumo/SuumoSearchURLBuilder.java @@ -0,0 +1,260 @@ +package xyz.kebigon.housesearch.browser.suumo; + +import xyz.kebigon.housesearch.domain.AnounceType; +import xyz.kebigon.housesearch.domain.Area; +import xyz.kebigon.housesearch.domain.SearchConditions; + +public class SuumoSearchURLBuilder +{ + private static final String SEARCH_URI = "https://suumo.jp/jj/bukken/ichiran/JJ012FC001"; + private static final String MIN_PRICE_FIELD = "kb"; + private static final String MAX_PRICE_FIELD = "kt"; + private static final String MIN_AGE_FIELD = "cnb"; + private static final String MAX_AGE_FIELD = "cn"; + private static final String MIN_LAND_SURFACE_FIELD = "tb"; + private static final String MAX_LAND_SURFACE_FIELD = "tt"; + private static final String MIN_HOUSE_SURFACE_FIELD = "hb"; + private static final String MAX_HOUSE_SURFACE_FIELD = "ht"; + private static final String MAX_WALK_TIME_TO_STATION_FIELD = "et"; + + public static String build(SearchConditions conditions) + { + final Area area = conditions.getArea(); + if (area == null) + throw new RuntimeException(); + + final AnounceType type = conditions.getType(); + if (type == null) + throw new RuntimeException(); + + final StringBuilder builder = new StringBuilder(SEARCH_URI); + builder.append("?ar=").append(area.getCode()); + builder.append("&bs=").append(type.getCode()); + builder.append("&pc=100"); // 100 results per page + builder.append("&kr=A"); // Buying the land with the house + + // Price + encodeMinPrice(builder, conditions.getMinPrice()); + encodeMaxPrice(builder, conditions.getMaxPrice()); + // Age + encodeMinAge(builder, conditions.getMinAge()); + encodeMaxAge(builder, conditions.getMaxAge()); + // LandSurface + encodeMinLandSurface(builder, conditions.getMinLandSurface()); + encodeMaxLandSurface(builder, conditions.getMaxLandSurface()); + // HouseSurface + encodeMinHouseSurface(builder, conditions.getMinHouseSurface()); + encodeMaxHouseSurface(builder, conditions.getMaxHouseSurface()); + // WalkTimeToStation + encodeMaxWalkTimeToStation(builder, conditions.getMaxWalkTimeToStation()); + + return builder.toString(); + } + + private static void encodeMinPrice(StringBuilder builder, Long minPrice) + { + if (minPrice == null || minPrice < 5000000) + return; // 下限なし + + builder.append('&').append(MIN_PRICE_FIELD).append('='); + + if (5000000 <= minPrice && minPrice < 10000000) + builder.append(500); // 500万円以上 + else if (10000000 <= minPrice && minPrice < 15000000) + builder.append(1000); // 1000万円以上 + else + builder.append(1500); // 1500万円以上 + } + + private static void encodeMaxPrice(StringBuilder builder, Long maxPrice) + { + if (maxPrice == null || 120000000 < maxPrice) + return; // 上限なし + + builder.append('&').append(MAX_PRICE_FIELD).append('='); + + if (100000000 < maxPrice && maxPrice <= 120000000) + builder.append(12000); // 1億2千万円未満 + else if (90000000 < maxPrice && maxPrice <= 100000000) + builder.append(10000); // 1億円未満 + else if (80000000 < maxPrice && maxPrice <= 90000000) + builder.append(9000); // 9000万円未満 + else if (75000000 < maxPrice && maxPrice <= 80000000) + builder.append(8000); // 8000万円未満 + else if (70000000 < maxPrice && maxPrice <= 75000000) + builder.append(7500); // 7500万円未満 + else if (65000000 < maxPrice && maxPrice <= 70000000) + builder.append(7000); // 7000万円未満 + else if (60000000 < maxPrice && maxPrice <= 65000000) + builder.append(6500); // 6500万円未満 + else if (55000000 < maxPrice && maxPrice <= 60000000) + builder.append(6000); // 6000万円未満 + else if (50000000 < maxPrice && maxPrice <= 55000000) + builder.append(5500); // 5500万円未満 + else if (45000000 < maxPrice && maxPrice <= 50000000) + builder.append(5000); // 5000万円未満 + else if (40000000 < maxPrice && maxPrice <= 45000000) + builder.append(4500); // 4500万円未満 + else if (35000000 < maxPrice && maxPrice <= 40000000) + builder.append(4000); // 4000万円未満 + else if (30000000 < maxPrice && maxPrice <= 35000000) + builder.append(3500); // 3500万円未満 + else if (25000000 < maxPrice && maxPrice <= 30000000) + builder.append(3000); // 3000万円未満 + else if (20000000 < maxPrice && maxPrice <= 25000000) + builder.append(2500); // 2500万円未満 + else if (15000000 < maxPrice && maxPrice <= 20000000) + builder.append(2000); // 2000万円未満 + else if (10000000 < maxPrice && maxPrice <= 15000000) + builder.append(1500); // 1500万円未満 + else if (5000000 < maxPrice && maxPrice <= 10000000) + builder.append(1000); // 1000万円未満 + else + builder.append(500); // 500万円未満 + } + + private static void encodeMinAge(StringBuilder builder, Integer minAge) + { + if (minAge == null || minAge < 3) + return; // 下限なし + + builder.append('&').append(MIN_AGE_FIELD).append('='); + + if (3 <= minAge && minAge < 5) + builder.append(3); // 3年以上 + else if (5 <= minAge && minAge < 7) + builder.append(5); // 5年以上 + else if (7 <= minAge && minAge < 10) + builder.append(7); // 7年以上 + else if (10 <= minAge && minAge < 15) + builder.append(10); // 10年以上 + else + builder.append(15); // 15年以上 + } + + private static void encodeMaxAge(StringBuilder builder, Integer maxAge) + { + if (maxAge == null || 30 < maxAge) + return; // 上限なし + + builder.append('&').append(MAX_AGE_FIELD).append('='); + + if (25 < maxAge && maxAge <= 30) + builder.append(30); // 30年以内 + else if (20 < maxAge && maxAge <= 25) + builder.append(25); // 25年以内 + else if (15 < maxAge && maxAge <= 20) + builder.append(20); // 20年以内 + else if (10 < maxAge && maxAge <= 15) + builder.append(15); // 15年以内 + else if (7 < maxAge && maxAge <= 10) + builder.append(10); // 10年以内 + else if (5 < maxAge && maxAge <= 7) + builder.append(7); // 7年以内 + else if (3 < maxAge && maxAge <= 5) + builder.append(5); // 5年以内 + else + builder.append(3); // 3年以内 + } + + private static void encodeMinLandSurface(StringBuilder builder, Integer minLandSurface) + { + encodeMinSurface(builder, MIN_LAND_SURFACE_FIELD, minLandSurface); + } + + private static void encodeMaxLandSurface(StringBuilder builder, Integer maxLandSurface) + { + encodeMaxSurface(builder, MAX_LAND_SURFACE_FIELD, maxLandSurface); + } + + private static void encodeMinHouseSurface(StringBuilder builder, Integer minHouseSurface) + { + encodeMinSurface(builder, MIN_HOUSE_SURFACE_FIELD, minHouseSurface); + } + + private static void encodeMaxHouseSurface(StringBuilder builder, Integer maxHouseSurface) + { + encodeMaxSurface(builder, MAX_HOUSE_SURFACE_FIELD, maxHouseSurface); + } + + private static void encodeMinSurface(StringBuilder builder, String field, Integer minLandSurface) + { + if (minLandSurface == null || minLandSurface < 60) + return; // 下限なし + + builder.append('&').append(field).append('='); + + if (60 <= minLandSurface && minLandSurface < 70) + builder.append(60); // 60m2以上 + else if (70 <= minLandSurface && minLandSurface < 80) + builder.append(70); // 70m2以上 + else if (80 <= minLandSurface && minLandSurface < 90) + builder.append(80); // 80m2以上 + else if (90 <= minLandSurface && minLandSurface < 100) + builder.append(90); // 90m2以上 + else if (100 <= minLandSurface && minLandSurface < 110) + builder.append(100); // 100m2以上 + else if (110 <= minLandSurface && minLandSurface < 120) + builder.append(110); // 110m2以上 + else if (120 <= minLandSurface && minLandSurface < 130) + builder.append(120); // 120m2以上 + else if (130 <= minLandSurface && minLandSurface < 140) + builder.append(130); // 130m2以上 + else if (140 <= minLandSurface && minLandSurface < 150) + builder.append(140); // 140m2以上 + else + builder.append(150); // 150m2以上 + } + + private static void encodeMaxSurface(StringBuilder builder, String field, Integer maxLandSurface) + { + if (maxLandSurface == null || 150 < maxLandSurface) + return; // 上限なし + + builder.append('&').append(field).append('='); + + if (140 < maxLandSurface && maxLandSurface <= 150) + builder.append(150); // 150m2未満 + else if (130 < maxLandSurface && maxLandSurface <= 140) + builder.append(140); // 140m2未満 + else if (120 < maxLandSurface && maxLandSurface <= 130) + builder.append(130); // 130m2未満 + else if (110 < maxLandSurface && maxLandSurface <= 120) + builder.append(120); // 120m2未満 + else if (100 < maxLandSurface && maxLandSurface <= 110) + builder.append(110); // 110m2未満 + else if (90 < maxLandSurface && maxLandSurface <= 100) + builder.append(100); // 100m2未満 + else if (80 < maxLandSurface && maxLandSurface <= 90) + builder.append(90); // 90m2未満 + else if (70 < maxLandSurface && maxLandSurface <= 80) + builder.append(80); // 80m2未満 + else if (60 < maxLandSurface && maxLandSurface <= 70) + builder.append(70); // 70m2未満 + else + builder.append(60); // 60m2未満 + } + + public static void encodeMaxWalkTimeToStation(StringBuilder builder, Integer maxWalkTimeToStation) + { + if (maxWalkTimeToStation == null || 20 < maxWalkTimeToStation) + return; // 指定なし + + builder.append('&').append(MAX_WALK_TIME_TO_STATION_FIELD).append('='); + + if (15 < maxWalkTimeToStation && maxWalkTimeToStation <= 20) + builder.append(20); // 20分以内 + else if (10 < maxWalkTimeToStation && maxWalkTimeToStation <= 15) + builder.append(15); // 15分以内 + else if (7 < maxWalkTimeToStation && maxWalkTimeToStation <= 10) + builder.append(10); // 10分以内 + else if (5 < maxWalkTimeToStation && maxWalkTimeToStation <= 7) + builder.append(7); // 7分以内 + else if (3 < maxWalkTimeToStation && maxWalkTimeToStation <= 5) + builder.append(5); // 5分以内 + else if (1 < maxWalkTimeToStation && maxWalkTimeToStation <= 3) + builder.append(3); // 3分以内 + else + builder.append(1); // 1分以内 + } +} diff --git a/src/main/java/xyz/kebigon/housesearch/browser/yahoo/transit/YahooTransitBrowser.java b/src/main/java/xyz/kebigon/housesearch/browser/yahoo/transit/YahooTransitBrowser.java @@ -0,0 +1,121 @@ +package xyz.kebigon.housesearch.browser.yahoo.transit; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.HashSet; + +import org.openqa.selenium.By; +import org.openqa.selenium.NoSuchElementException; +import org.openqa.selenium.WebElement; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; + +import xyz.kebigon.housesearch.browser.Browser; +import xyz.kebigon.housesearch.domain.Route; +import xyz.kebigon.housesearch.file.RoutesCache; + +public class YahooTransitBrowser extends Browser +{ + private final RoutesCache cache; + + public YahooTransitBrowser() throws JsonParseException, JsonMappingException, IOException + { + final File file = new File("cache.json"); + cache = RoutesCache.load(file); + } + + @Override + public void close() throws IOException + { + super.close(); + + final File file = new File("cache.json"); + cache.save(file); + } + + public Collection<Route> search(String from, String to) + { + final String cacheKey = from + "-" + to; + + Collection<Route> routes = cache.get(cacheKey); + if (routes != null) + return routes; + + final String url = YahooTransitSearchURLBuilder.build(from, to); + + navigateTo(url); + + routes = new HashSet<Route>(); + + try + { + for (final WebElement element : findElements("//ul[@class='routeList']/li/dl/dd/ul")) + routes.add(createRoute(from, to, element)); + + click("//ul[@id='tabflt']/li/a[@data-rapid_p='2']"); + + for (final WebElement element : findElements("//ul[@class='routeList']/li/dl/dd/ul")) + routes.add(createRoute(from, to, element)); + + click("//ul[@id='tabflt']/li/a[@data-rapid_p='3']"); + + for (final WebElement element : findElements("//ul[@class='routeList']/li/dl/dd/ul")) + routes.add(createRoute(from, to, element)); + } + catch (final NoSuchElementException e) + { + } + + cache.put(cacheKey, routes); + + return routes; + } + + private static final By TIME_XPATH = By.xpath("./li[@class='time']/span[@class='small']"); + private static final By FARE_XPATH = By.xpath("(./li[@class='fare']/div[@class='mark'] | ./li[@class='fare'])[1]"); + private static final By TRANSFER_XPATH = By.xpath("(./li[@class='transfer']/div[@class='mark'] | ./li[@class='transfer'])[1]"); + + private static Route createRoute(String from, String to, WebElement element) + { + final int time = parseTime(element.findElement(TIME_XPATH).getText()); + final int fare = parseFare(element.findElement(FARE_XPATH).getText()); + final int transfer = parseTransfert(element.findElement(TRANSFER_XPATH).getText()); + + return new Route(from, to, time, fare, transfer); + } + + // 1時間47分 -> 107 + private static int parseTime(String timeString) + { + final int hourIndex = timeString.indexOf("時間"); + final int minuteIndex = timeString.indexOf('分'); + + if (hourIndex != -1) + { + final int hour = Integer.parseInt(timeString.substring(0, hourIndex)); + final int minutes = Integer.parseInt(timeString.substring(hourIndex + 2, minuteIndex)); + + return hour * 60 + minutes; + } + else + return Integer.parseInt(timeString.substring(0, minuteIndex)); + } + + // 1,329円 -> 1329 + private static int parseFare(String fareString) + { + return Integer.parseInt(fareString.replace(",", "").replace("円", "")); + } + + // 乗換：2回 + // 0回 + private static int parseTransfert(String transfertString) + { + if (transfertString.startsWith("乗換：")) + return Integer.parseInt(transfertString.substring(3, transfertString.length() - 1)); + else + return Integer.parseInt(transfertString.substring(0, transfertString.length() - 1)); + } +} diff --git a/src/main/java/xyz/kebigon/housesearch/browser/yahoo/transit/YahooTransitSearchURLBuilder.java b/src/main/java/xyz/kebigon/housesearch/browser/yahoo/transit/YahooTransitSearchURLBuilder.java @@ -0,0 +1,35 @@ +package xyz.kebigon.housesearch.browser.yahoo.transit; + +import java.time.LocalDate; + +public class YahooTransitSearchURLBuilder +{ + private static final String SEARCH_URI = "https://transit.yahoo.co.jp/search/result"; + + public static String build(String from, String to) + { + LocalDate today = LocalDate.now(); + switch (today.getDayOfWeek()) + { + case SUNDAY: + today = today.plusDays(1); + break; + case SATURDAY: + today = today.plusDays(2); + break; + default: + break; + } + + final StringBuilder builder = new StringBuilder(SEARCH_URI); + builder.append("?from=").append(from); + builder.append("&to=").append(to); + builder.append("&y=").append(today.getYear()); + builder.append("&m=").append(String.format("%02d", today.getMonthValue())); + builder.append("&d=").append(String.format("%02d", today.getDayOfMonth())); + builder.append("&hh=10&m2=0&m1=0"); // 10:00 + builder.append("&type=4"); + + return builder.toString(); + } +} diff --git a/src/main/java/xyz/kebigon/housesearch/domain/AnounceType.java b/src/main/java/xyz/kebigon/housesearch/domain/AnounceType.java @@ -0,0 +1,18 @@ +package xyz.kebigon.housesearch.domain; + +public enum AnounceType +{ + NEW_CONDO("010"), USED_CONDO("011"), NEW_HOUSE("020"), USED_HOUSE("021"), RENT("040"); + + private final String code; + + private AnounceType(String code) + { + this.code = code; + } + + public String getCode() + { + return code; + } +} diff --git a/src/main/java/xyz/kebigon/housesearch/domain/Area.java b/src/main/java/xyz/kebigon/housesearch/domain/Area.java @@ -0,0 +1,18 @@ +package xyz.kebigon.housesearch.domain; + +public enum Area +{ + HOKKAIDO("010"), TOHOKU("020"), KANTO("030"), KOSHINETSU("040"), TOKAI("050"), KANSAI("060"), SHIKOKU("070"), CHUGOKU("080"), KYUSHU("090"); + + private final String code; + + private Area(String code) + { + this.code = code; + } + + public String getCode() + { + return code; + } +} diff --git a/src/main/java/xyz/kebigon/housesearch/domain/Posting.java b/src/main/java/xyz/kebigon/housesearch/domain/Posting.java @@ -0,0 +1,67 @@ +package xyz.kebigon.housesearch.domain; + +import lombok.Data; + +@Data +public class Posting +{ + /** + * URL + */ + private final String url; + + /** + * Price (in yens) + */ + private final Long price; + + /** + * Age (in years) + */ + private final Integer age; + + /** + * Land surface (in squared meters) + */ + private final Double landSurface; + + /** + * House surface (in squared meters) + */ + private final Double houseSurface; + + /** + * Time to go to the closest station (in minutes) + */ + private final Integer walkTimeToStation; + + /** + * Closest station + */ + private final String station; + + /** + * Fastest route in the ones compared by spring expression + */ + private Route fastestRoute; + + /** + * Cheapest route in the ones compared by spring expression + */ + private Route cheapestRoute; + + /** + * Easiest route in the ones compared by spring expression + */ + private Route easiestRoute; + + public void updateRoutes(Route route) + { + if (fastestRoute == null || route.getTime() < fastestRoute.getTime()) + fastestRoute = route; + if (cheapestRoute == null || route.getFare() < cheapestRoute.getFare()) + cheapestRoute = route; + if (easiestRoute == null || route.getTransfer() < easiestRoute.getTransfer()) + easiestRoute = route; + } +} diff --git a/src/main/java/xyz/kebigon/housesearch/domain/Route.java b/src/main/java/xyz/kebigon/housesearch/domain/Route.java @@ -0,0 +1,97 @@ +package xyz.kebigon.housesearch.domain; + +import java.util.Comparator; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@AllArgsConstructor +@Data +@NoArgsConstructor +public class Route +{ + /** + * the station of departure + */ + private String from; + + /** + * the station of arrival + */ + private String to; + + /** + * time in minutes between the two stations + */ + private int time; + + /** + * price in yens between the two stations + */ + private int fare; + + /** + * number of transfers between the two stations + */ + private int transfer; + + /** + * Time > Fare > Transfer comparator + */ + public static final Comparator<Route> TIME_COMPARATOR = new Comparator<Route>() + { + @Override + public int compare(Route route1, Route route2) + { + int result; + if ((result = Integer.compare(route1.time, route2.time)) != 0) + return result; + else if ((result = Integer.compare(route1.fare, route2.fare)) != 0) + return result; + else + return Integer.compare(route1.transfer, route2.transfer); + } + }; + + /** + * Fare > Transfer > Time comparator + */ + public static final Comparator<Route> FARE_COMPARATOR = new Comparator<Route>() + { + @Override + public int compare(Route route1, Route route2) + { + int result; + if ((result = Integer.compare(route1.fare, route2.fare)) != 0) + return result; + else if ((result = Integer.compare(route1.transfer, route2.transfer)) != 0) + return result; + else + return Integer.compare(route1.time, route2.time); + } + }; + + /** + * Transfer > Time > Fare comparator + */ + public static final Comparator<Route> TRANSFER_COMPARATOR = new Comparator<Route>() + { + @Override + public int compare(Route route1, Route route2) + { + int result; + if ((result = Integer.compare(route1.transfer, route2.transfer)) != 0) + return result; + else if ((result = Integer.compare(route1.time, route2.time)) != 0) + return result; + else + return Integer.compare(route1.fare, route2.fare); + } + }; + + /** + * Route to be used when no routes are found + */ + public static final Route IMPOSSIBLE_ROUTE = new Route(null, null, 9999, 9999, 9999); +} diff --git a/src/main/java/xyz/kebigon/housesearch/domain/SearchConditions.java b/src/main/java/xyz/kebigon/housesearch/domain/SearchConditions.java @@ -0,0 +1,96 @@ +package xyz.kebigon.housesearch.domain; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Properties; + +import org.springframework.util.StringUtils; + +import lombok.Data; + +@Data +public class SearchConditions +{ + private Area area; + private AnounceType type; + private Long minPrice; + private Long maxPrice; + private Integer minAge; + private Integer maxAge; + private Integer minLandSurface; + private Integer maxLandSurface; + private Integer minHouseSurface; + private Integer maxHouseSurface; + private Integer maxWalkTimeToStation; + private String expression; + + public static SearchConditions load() throws IOException + { + final Properties properties = new Properties(); + properties.load(SearchConditions.class.getClassLoader().getResourceAsStream("search-conditions.cfg")); + + final SearchConditions conditions = new SearchConditions(); + conditions.setArea(areaProp(properties, "area")); + conditions.setType(anounceTypeProp(properties, "type")); + conditions.setMinPrice(longProp(properties, "price.min")); + conditions.setMaxPrice(longProp(properties, "price.max")); + conditions.setMinAge(intProp(properties, "age.min")); + conditions.setMaxAge(intProp(properties, "age.max")); + conditions.setMinLandSurface(intProp(properties, "surface.land.min")); + conditions.setMaxLandSurface(intProp(properties, "surface.land.max")); + conditions.setMinHouseSurface(intProp(properties, "surface.house.min")); + conditions.setMaxHouseSurface(intProp(properties, "surface.house.max")); + conditions.setMaxWalkTimeToStation(intProp(properties, "station.walktime.max")); + conditions.setExpression(expressionProp(properties, "expression.file")); + return conditions; + } + + private static Area areaProp(Properties properties, String key) + { + final String property = properties.getProperty(key); + return StringUtils.isEmpty(property) ? null : Area.valueOf(property); + } + + private static AnounceType anounceTypeProp(Properties properties, String key) + { + final String property = properties.getProperty(key); + return StringUtils.isEmpty(property) ? null : AnounceType.valueOf(property); + } + + private static Integer intProp(Properties properties, String key) + { + final String property = properties.getProperty(key); + return StringUtils.isEmpty(property) ? null : Integer.parseInt(property); + } + + private static Long longProp(Properties properties, String key) + { + final String property = properties.getProperty(key); + return StringUtils.isEmpty(property) ? null : Long.parseLong(property); + } + + private static String expressionProp(Properties properties, String key) throws IOException + { + final String property = properties.getProperty(key); + if (StringUtils.isEmpty(property)) + return null; + + String expression = ""; + try (final BufferedReader reader = new BufferedReader(new InputStreamReader(SearchConditions.class.getClassLoader().getResourceAsStream(property)))) + { + String line; + while ((line = reader.readLine()) != null) + { + line = line.trim(); + if (line.isEmpty()) + continue; + if (line.startsWith("#")) + continue; + expression += line; + } + } + + return expression; + } +} diff --git a/src/main/java/xyz/kebigon/housesearch/domain/SearchConditionsValidator.java b/src/main/java/xyz/kebigon/housesearch/domain/SearchConditionsValidator.java @@ -0,0 +1,149 @@ +package xyz.kebigon.housesearch.domain; + +import org.springframework.expression.Expression; +import org.springframework.expression.ExpressionParser; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.expression.spel.support.StandardEvaluationContext; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import xyz.kebigon.housesearch.ApplicationContext; + +@AllArgsConstructor +@Slf4j +public class SearchConditionsValidator +{ + public static boolean validateBasicConditions(Posting posting, SearchConditions conditions) + { + final String url = posting.getUrl(); + if (url == null) + { + log.warn("Missing url for {}", posting); + return false; + } + + final Long price = posting.getPrice(); + if (price == null) + { + log.warn("Missing price for {}", posting); + return false; + } + if (conditions.getMinPrice() != null && price < conditions.getMinPrice()) + return false; + if (conditions.getMaxPrice() != null && price > conditions.getMaxPrice()) + return false; + + final Integer age = posting.getAge(); + if (age == null) + { + log.warn("Missing age for {}", posting); + return false; + } + if (conditions.getMinAge() != null && age < conditions.getMinAge()) + return false; + if (conditions.getMaxAge() != null && age > conditions.getMaxAge()) + return false; + + final Double landSurface = posting.getLandSurface(); + if (landSurface == null) + { + log.warn("Missing landSurface for {}", posting); + return false; + } + if (conditions.getMinLandSurface() != null && landSurface < conditions.getMinLandSurface()) + return false; + if (conditions.getMaxLandSurface() != null && landSurface > conditions.getMaxLandSurface()) + return false; + + final Double houseSurface = posting.getHouseSurface(); + if (houseSurface == null) + { + log.warn("Missing houseSurface for {}", posting); + return false; + } + if (conditions.getMinHouseSurface() != null && houseSurface < conditions.getMinHouseSurface()) + return false; + if (conditions.getMaxHouseSurface() != null && houseSurface > conditions.getMaxHouseSurface()) + return false; + + final Integer walkTimeToStation = posting.getWalkTimeToStation(); + if (walkTimeToStation == null) + { + log.warn("Missing walkTimeToStation for {}", posting); + return false; + } + if (conditions.getMaxWalkTimeToStation() != null && walkTimeToStation > conditions.getMaxWalkTimeToStation()) + return false; + + final String station = posting.getStation(); + if (station == null) + { + log.warn("Missing station for {}", posting); + return false; + } + + return true; + } + + public static boolean validateExpression(Posting posting, SearchConditions conditions) + { + final ExpressionParser expressionParser = new SpelExpressionParser(); + + final StandardEvaluationContext context = new StandardEvaluationContext(); + context.setVariable("property", posting); + + try + { + context.registerFunction("timeToStation", + SearchConditionsValidator.class.getDeclaredMethod("timeToStation", new Class[] { Posting.class, String.class })); + context.registerFunction("fareToStation", + SearchConditionsValidator.class.getDeclaredMethod("fareToStation", new Class[] { Posting.class, String.class })); + context.registerFunction("transferToStation", + SearchConditionsValidator.class.getDeclaredMethod("transferToStation", new Class[] { Posting.class, String.class })); + } + catch (NoSuchMethodException | SecurityException e) + { + e.printStackTrace(); + throw new RuntimeException(e); + } + + final Expression expression = expressionParser.parseExpression(conditions.getExpression()); + return expression.getValue(context, posting, Boolean.class); + } + + @SuppressWarnings("unused") + private static int timeToStation(Posting posting, String station) + { + final Route route = ApplicationContext.getYahooTransitBrowser() // + .search(posting.getStation(), station).stream() // + .min(Route.TIME_COMPARATOR).orElse(Route.IMPOSSIBLE_ROUTE); + + posting.updateRoutes(route); + + return route.getTime() + posting.getWalkTimeToStation(); + } + + @SuppressWarnings("unused") + private static int fareToStation(Posting posting, String station) + { + final Route route = ApplicationContext.getYahooTransitBrowser() // + .search(posting.getStation(), station).stream() // + .min(Route.FARE_COMPARATOR).orElse(Route.IMPOSSIBLE_ROUTE); + + posting.updateRoutes(route); + + return route.getFare(); + } + + @SuppressWarnings("unused") + private static int transferToStation(Posting posting, String station) + { + final Route route = ApplicationContext.getYahooTransitBrowser() // + .search(posting.getStation(), station).stream() // + .min(Route.TRANSFER_COMPARATOR).orElse(Route.IMPOSSIBLE_ROUTE); + + posting.updateRoutes(route); + + return route.getTransfer(); + } +} diff --git a/src/main/java/xyz/kebigon/housesearch/file/RoutesCache.java b/src/main/java/xyz/kebigon/housesearch/file/RoutesCache.java @@ -0,0 +1,55 @@ +package xyz.kebigon.housesearch.file; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +import com.fasterxml.jackson.core.JsonGenerationException; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import xyz.kebigon.housesearch.domain.Route; + +@Slf4j +@Data +@NoArgsConstructor +public class RoutesCache +{ + private Map<String, Collection<Route>> cache = new HashMap<>(); + + public Collection<Route> get(String cacheKey) + { + return cache.get(cacheKey); + } + + public void put(String cacheKey, Collection<Route> routes) + { + cache.put(cacheKey, routes); + } + + public void save(File file) throws JsonGenerationException, JsonMappingException, IOException + { + log.info("Saving {} routes to {}", cache.size(), file); + final ObjectMapper mapper = new ObjectMapper(); + mapper.writeValue(file, this); + } + + public static RoutesCache load(File file) throws JsonParseException, JsonMappingException, IOException + { + if (file.exists()) + { + log.info("Loading routes cache from {}", file); + + final ObjectMapper mapper = new ObjectMapper(); + return mapper.readValue(file, RoutesCache.class); + } + else + return new RoutesCache(); + } +} diff --git a/src/main/java/xyz/kebigon/housesearch/file/SearchArchive.java b/src/main/java/xyz/kebigon/housesearch/file/SearchArchive.java @@ -0,0 +1,59 @@ +package xyz.kebigon.housesearch.file; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.Closeable; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Writer; +import java.util.Collection; +import java.util.HashSet; + +import lombok.extern.slf4j.Slf4j; +import xyz.kebigon.housesearch.domain.Posting; + +@Slf4j +public class SearchArchive implements Closeable +{ + private static final File ARCHIVE_FILE = new File("archive"); + + private final Collection<String> urls = new HashSet<String>(); + + public SearchArchive() throws IOException + { + if (ARCHIVE_FILE.exists()) + { + try (final BufferedReader reader = new BufferedReader(new FileReader(ARCHIVE_FILE))) + { + String line; + while ((line = reader.readLine()) != null) + urls.add(line); + } + + log.info("Loaded {} urls from {}", urls.size(), ARCHIVE_FILE.getAbsolutePath()); + } + } + + public boolean filter(Posting property) + { + if (urls.contains(property.getUrl())) + return false; + + urls.add(property.getUrl()); + return true; + } + + @Override + public void close() throws IOException + { + try (final Writer writer = new BufferedWriter(new FileWriter(ARCHIVE_FILE))) + { + for (final String url : urls) + writer.write(url + '\n'); + } + + log.info("Saved {} urls to {}", urls.size(), ARCHIVE_FILE.getAbsolutePath()); + } +} diff --git a/src/main/java/xyz/kebigon/housesearch/mail/EmailSender.java b/src/main/java/xyz/kebigon/housesearch/mail/EmailSender.java @@ -0,0 +1,132 @@ +package xyz.kebigon.housesearch.mail; + +import java.io.IOException; +import java.util.Collection; +import java.util.Properties; + +import javax.mail.Session; + +import org.apache.commons.mail.Email; +import org.apache.commons.mail.EmailException; +import org.apache.commons.mail.HtmlEmail; + +import xyz.kebigon.housesearch.HouseSearchApplication; +import xyz.kebigon.housesearch.domain.Posting; +import xyz.kebigon.housesearch.domain.Route; + +public class EmailSender +{ + private final Session mailSession; + + public EmailSender() throws IOException + { + final Properties mailProperties = new Properties(); + mailProperties.load(HouseSearchApplication.class.getClassLoader().getResourceAsStream("mail.cfg")); + mailSession = Session.getInstance(mailProperties); + } + + public void send(Collection<Posting> postings) throws EmailException + { + // Send email separately + if (postings.size() <= 3) + { + for (final Posting posting : postings) + { + final StringBuilder builder = new StringBuilder(); + appendPosting(posting, builder); + send(posting.getAge() + "yo house found near " + posting.getStation() + " station for " + (posting.getPrice() / 10000) + "万円", + builder.toString()); + } + } + // Send one email with all results + else + { + final StringBuilder builder = new StringBuilder(); + for (final Posting posting : postings) + appendPosting(posting, builder); + send(postings.size() + " houses found", builder.toString()); + } + } + + private void send(String title, String content) throws EmailException + { + final Email email = new HtmlEmail(); + email.setMailSession(mailSession); + email.setSubject(title); + email.setCharset("UTF-8"); + email.setMsg(content); + + email.setFrom(mailSession.getProperty("housesearch.mail.to").trim()); + for (final String address : mailSession.getProperty("housesearch.mail.to").split(",")) + email.addTo(address.trim()); + + email.send(); + } + + private static void appendPosting(Posting posting, StringBuilder builder) + { + builder.append("<p><a href=\"").append(posting.getUrl()).append("\">"); + builder.append(posting.getAge()).append("yo house near ").append(posting.getStation()).append(" station for ").append(posting.getPrice() / 10000) + .append("万円"); + builder.append("</a>"); + builder.append("<br>House surface: ").append(Math.round(posting.getHouseSurface())).append("m2 / ") + .append(Math.round(posting.getHouseSurface() * 0.3025)).append("坪"); + builder.append("<br>Land surface: ").append(Math.round(posting.getLandSurface())).append("m2 / ").append(Math.round(posting.getLandSurface() * 0.3025)) + .append("坪"); + + appendRoutes(posting, builder); + + builder.append("</p>"); + } + + private static void appendRoutes(Posting posting, StringBuilder builder) + { + final Route fastestRoute = posting.getFastestRoute(); + final Route cheapestRoute = posting.getCheapestRoute(); + final Route easiestRoute = posting.getEasiestRoute(); + + // No route have been analyzed + if (fastestRoute == null && cheapestRoute == null && easiestRoute == null) + return; + + builder.append("<br>Routes: "); + + if (fastestRoute == cheapestRoute) + if (fastestRoute == easiestRoute) + appendRoute("Best", posting, fastestRoute, builder); + else + { + appendRoute("Fastest & Cheapest", posting, fastestRoute, builder); + appendRoute("Easiest", posting, easiestRoute, builder); + } + else if (fastestRoute == easiestRoute) + { + appendRoute("Fastest & Easiest", posting, fastestRoute, builder); + appendRoute("Cheapest", posting, cheapestRoute, builder); + } + else + { + appendRoute("Fastest", posting, fastestRoute, builder); + + if (cheapestRoute == easiestRoute) + appendRoute("Cheapest & Easiest", posting, cheapestRoute, builder); + else + { + appendRoute("Cheapest", posting, cheapestRoute, builder); + appendRoute("Easiest", posting, easiestRoute, builder); + } + } + } + + private static void appendRoute(String label, Posting property, Route route, StringBuilder builder) + { + if (route == null) + return; + + builder.append("<br>- ").append(label).append(": "); + builder.append(route.getTime() + property.getWalkTimeToStation()).append("min to ").append(route.getTo()); + builder.append(" (walk: ").append(property.getWalkTimeToStation()).append("min, train: ").append(route.getTime()).append("min)"); + builder.append(" / ").append(route.getFare()).append("円"); + builder.append(" / ").append(route.getTransfer()).append(" transfer"); + } +} diff --git a/src/main/packaged-resources/cfg/log4j2.xml b/src/main/packaged-resources/cfg/log4j2.xml @@ -0,0 +1,19 @@ +<?xml version="1.0" encoding="UTF-8"?> +<Configuration status="WARN"> + <Appenders> + <Console name="Console" target="SYSTEM_OUT"> + <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n" /> + </Console> + </Appenders> + <Loggers> + <Logger name="com.gargoylesoftware.htmlunit" level="error" additivity="false"> + <AppenderRef ref="Console" /> + </Logger> + <Logger name="org.apache.http" level="error" additivity="false"> + <AppenderRef ref="Console" /> + </Logger> + <Root level="trace"> + <AppenderRef ref="Console" /> + </Root> + </Loggers> +</Configuration> +\ No newline at end of file diff --git a/src/main/packaged-resources/cfg/mail.cfg b/src/main/packaged-resources/cfg/mail.cfg @@ -0,0 +1,10 @@ +mail.smtp.host= +mail.smtp.port=465 +mail.smtp.ssl.enable=true + +mail.smtp.auth=true +mail.smtp.user= +mail.smtp.password= + +housesearch.mail.from= +housesearch.mail.to= +\ No newline at end of file diff --git a/src/main/packaged-resources/cfg/search-conditions.cfg b/src/main/packaged-resources/cfg/search-conditions.cfg @@ -0,0 +1,22 @@ +area=KANTO +type=USED_HOUSE + +# Price of the house (in yens) +price.min= +price.max=20000000 + +# Age of the house (in years) +age.min= +age.max=20 + +# Surface of the house/land (in squared meters) +surface.land.min=100 +surface.land.max= +surface.house.min=80 +surface.house.max= + +# Time to go to the nearest station by foot (in minutes) +station.walktime.max=15 + +# More complex conditions in form of a spring expression +expression.file=search-expression.cfg diff --git a/src/main/packaged-resources/cfg/search-expression.cfg b/src/main/packaged-resources/cfg/search-expression.cfg @@ -0,0 +1,24 @@ + +# At least one good route +( + # Fast route + + (#timeToStation(#property, '東京駅') <= 40 && #fareToStation(#property, '東京駅') <= 750 && #transferToStation(#property, '東京駅') <= 1) + || (#timeToStation(#property, '三越前') <= 40 && #fareToStation(#property, '三越前') <= 750 && #transferToStation(#property, '三越前') <= 1) + || (#timeToStation(#property, '大手町(東京都)駅') <= 40 && #fareToStation(#property, '大手町(東京都)駅') <= 750 && #transferToStation(#property, '大手町(東京都)駅') <= 1) + + # Cheap route + + || (#timeToStation(#property, '東京駅') <= 60 && #fareToStation(#property, '東京駅') <= 500 && #transferToStation(#property, '東京駅') <= 1) + || (#timeToStation(#property, '三越前') <= 60 && #fareToStation(#property, '三越前') <= 500 && #transferToStation(#property, '三越前') <= 1) + || (#timeToStation(#property, '大手町(東京都)駅') <= 60 && #fareToStation(#property, '大手町(東京都)駅') <= 500 && #transferToStation(#property, '大手町(東京都)駅') <= 1) + + # Direct rotue + + || (#timeToStation(#property, '東京駅') <= 60 && #fareToStation(#property, '東京駅') <= 750 && #transferToStation(#property, '東京駅') == 0) + || (#timeToStation(#property, '三越前') <= 60 && #fareToStation(#property, '三越前') <= 750 && #transferToStation(#property, '三越前') == 0) + || (#timeToStation(#property, '大手町(東京都)駅') <= 60 && #fareToStation(#property, '大手町(東京都)駅') <= 750 && #transferToStation(#property, '大手町(東京都)駅') == 0) +) + +# Mistaken with Tokyo's Akasaka +&& !url.startsWith('https://suumo.jp/chukoikkodate/gumma/sc_maebashi') +\ No newline at end of file diff --git a/src/test/java/xyz/kebigon/suumo/browser/SearchURLBuilderTest.java b/src/test/java/xyz/kebigon/suumo/browser/SearchURLBuilderTest.java @@ -0,0 +1,47 @@ +package xyz.kebigon.suumo.browser; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import xyz.kebigon.housesearch.browser.suumo.SuumoSearchURLBuilder; +import xyz.kebigon.housesearch.domain.AnounceType; +import xyz.kebigon.housesearch.domain.Area; +import xyz.kebigon.housesearch.domain.SearchConditions; + +public class SearchURLBuilderTest +{ + @Test + public void testSearch() + { + final SearchConditions conditions = new SearchConditions(); + conditions.setArea(Area.KANTO); + conditions.setType(AnounceType.USED_HOUSE); + conditions.setMaxPrice(20000000l); + conditions.setMaxAge(10); + conditions.setMinLandSurface(100); + conditions.setMinHouseSurface(80); + + assertEquals("https://suumo.jp/jj/bukken/ichiran/JJ012FC001?ar=030&bs=021&pc=100&kr=A&kt=2000&cn=10&tb=100&hb=80", + SuumoSearchURLBuilder.build(conditions)); + } + + @Test + public void testExactMatch() + { + final SearchConditions conditions = new SearchConditions(); + conditions.setArea(Area.KANTO); + conditions.setType(AnounceType.USED_HOUSE); + conditions.setMinPrice(6780000l); + conditions.setMaxPrice(12340000l); + conditions.setMinAge(11); + conditions.setMaxAge(27); + conditions.setMinLandSurface(82); + conditions.setMaxLandSurface(125); + conditions.setMinHouseSurface(73); + conditions.setMaxHouseSurface(86); + + assertEquals("https://suumo.jp/jj/bukken/ichiran/JJ012FC001?ar=030&bs=021&pc=100&kr=A&kb=500&kt=1500&cnb=10&cn=30&tb=80&tt=130&hb=70&ht=90", + SuumoSearchURLBuilder.build(conditions)); + } +}