Changeset 1385 in algoquant

Timestamp:

3/15/2013 8:35:58 PM (11 months ago)

Author:

kenyiu

Message:

CHG: Changed the default holidays data folder to ./log/holidays, and the property name for the setting to DATA_HOLIDAYS.

Location:

core

Files:

• src/main/java/com/numericalmethod/algoquant/config/ConfigProperties.java (modified) (1 diff)
• src/main/java/com/numericalmethod/algoquant/data/calendar/holiday/HolidaysFromFile.java (modified) (8 diffs)
• src/main/resources/config/data.properties (modified) (1 diff)
• src/main/resources/config/haksunli/haksunli.properties (modified) (1 diff)
• src/test/java/com/numericalmethod/algoquant/config/AlgoQuantConfigTest.java (modified) (5 diffs)
• src/test/java/com/numericalmethod/algoquant/data/calendar/holiday/HolidaysFromFileTest.java (modified) (8 diffs)
• test-data/config/data.properties (modified) (1 diff)
• test-data/config/data2.properties (modified) (1 diff)

core/src/main/java/com/numericalmethod/algoquant/config/ConfigProperties.java

@@ -49 +49 @@
      * The data folder location that stores holiday data files.
      */
-    HOLIDAY_DIR("HOLIDAY_DIR");
+    DATA_HOLIDAYS("DATA_HOLIDAYS");
     private final String propertyName;
 

core/src/main/java/com/numericalmethod/algoquant/data/calendar/holiday/HolidaysFromFile.java

@@ -48 +48 @@
 public class HolidaysFromFile implements Holidays {
 
-    private static final String HOLIDAY_DIR = AlgoQuantConfig.getConfiguration().getString(
-        ConfigProperties.HOLIDAY_DIR.getPropertyName(), "./log/holidays");
+    private static final Logger LOGGER = getClassLogger();
+    public static final String DEFAULT_HOLIDAYS_DIRNAME = "./log/holidays";
+    private static final String HOLIDAY_DIRNAME = AlgoQuantConfig.getConfiguration().getString(
+        ConfigProperties.DATA_HOLIDAYS.getPropertyName(), DEFAULT_HOLIDAYS_DIRNAME);
     private static final String MATCHING_PATTERN = "^%s(\\d{4})-(\\d{2})-(\\d{2})~(\\d{4})-(\\d{2})-(\\d{2}).csv.zip$";
     private static final String DATETIME_FORMAT = "yyyy-MM-dd";
     private static final DateTimeFormatter DATETIME_FORMATTER = DateTimeFormat.forPattern(DATETIME_FORMAT);
-    private static final Logger logger = getClassLogger();
     private final ExchangeIndex exchangeIndex;
+    private final boolean updateFromWeb;
     private String holidayFileName = null;
     private Set<DateMidnight> holidaysSet = null;
 
     public HolidaysFromFile(ExchangeIndex exchangeIndex) {
+        this(exchangeIndex, true);
+    }
+
+    public HolidaysFromFile(ExchangeIndex exchangeIndex, boolean updateFromWeb) {
         this.exchangeIndex = exchangeIndex;
+        this.updateFromWeb = updateFromWeb;
     }
 
     private boolean createHolidayFile(DateMidnight endDate) {
+        if (!updateFromWeb) {
+            LOGGER.info(String.format("holiday file is not updated from web for %s on %s",
+                                      exchangeIndex, endDate));
+            return false;
+        }
 
         try {
@@ -72 +84 @@
             holidayFileName = getFileName(exchangeIndex, beginDate, endDate);
 
-            saveHolidaysFile(weekdayHolidays, HOLIDAY_DIR, holidayFileName);
+            saveHolidaysFile(weekdayHolidays, HOLIDAY_DIRNAME, holidayFileName);
             holidaysSet = getHolidaysFromFile();
 
         } catch (Exception ex) {
-            logger.info(String.format("Error updating holiday file, reason = %s", ex.getMessage()));
+            LOGGER.error(String.format(
+                "Error updating holiday file, reason = %s", ex.getMessage()));
             return false;
         }
@@ -95 +108 @@
 
     private boolean isHolidayFileExist() {
-        File path = new File(HOLIDAY_DIR);
+        File path = new File(HOLIDAY_DIRNAME);
         if (!path.exists()) {
             return false;
@@ -103 +116 @@
             @Override
             public boolean accept(File dir, String name) {
-                return Pattern.matches(String.format(MATCHING_PATTERN, getFilePrefix(exchangeIndex)), name);
+                return Pattern.matches(
+                    String.format(MATCHING_PATTERN, getFilePrefix(exchangeIndex)), name);
             }
         });
@@ -116 +130 @@
 
     private boolean isHolidayFileUpdate(DateMidnight queryDate) {
-
         if (holidayFileName == null) {
             return false;
         }
 
-        Pattern pattern = Pattern.compile(String.format(MATCHING_PATTERN, getFilePrefix(exchangeIndex)));
+        Pattern pattern = Pattern.compile(
+            String.format(MATCHING_PATTERN, getFilePrefix(exchangeIndex)));
         Matcher matcher = pattern.matcher(holidayFileName);
         matcher.matches();
@@ -138 +152 @@
 
         if (queryDate.isBefore(fromDate) || queryDate.isAfter(toDate)) {
-            new File(String.format("%s/%s", HOLIDAY_DIR, holidayFileName)).delete();
+            new File(String.format("%s/%s", HOLIDAY_DIRNAME, holidayFileName)).delete();
             holidayFileName = null;
             return false;
@@ -152 +166 @@
         if (!isQueryDateInRange(queryDateMidnight)) {
             // If query date outside data file range, update is not success
-            logger.warn(String.format("Query date (%s) outside data file range (%s): %s ~ %s",
+            LOGGER.warn(String.format("Query date (%s) outside data file range (%s): %s ~ %s",
                                       queryDateMidnight.toString(DATETIME_FORMAT),
                                       getFilePrefix(exchangeIndex),
@@ -183 +197 @@
 
         HolidayCsvZipFileReader reader;
+        String fileLocation = String.format("%s/%s", HOLIDAY_DIRNAME, holidayFileName);
         try {
-            reader = new HolidayCsvZipFileReader(
-                String.format("%s/%s", HOLIDAY_DIR, holidayFileName));
+            reader = new HolidayCsvZipFileReader(fileLocation);
         } catch (Exception ex) {
-            logger.info(String.format("Error reading holiday file: %s", holidayFileName));
+            LOGGER.info(String.format("Error reading holiday file: %s", fileLocation));
             return holidays;
         }

core/src/main/resources/config/data.properties

@@ -5 +5 @@
 DATA_YAHOO = ${DATA_FOLDER}/yahoo
 DATA_SGX = ${DATA_FOLDER}/raw/sgx
-HOLIDAY_DIR = ./resources/holidays
+DATA_HOLIDAYS = ./log/holidays

core/src/main/resources/config/haksunli/haksunli.properties

@@ -7 +7 @@
 
 #These are the compustat files of hong kong stock data.
-COMPUSTAT_HONGKONG_COMPANY = C:/Users/Haksun Li/Documents/Businesses/Numerical Method/data/raw/compustat/hongkong/company.csv.zip
-COMPUSTAT_HONGKONG_ACCOUNTING = C:/Users/Haksun Li/Documents/Businesses/Numerical Method/data/raw/compustat/hongkong/monthlyAccounting.1995.2010.csv.zip
-COMPUSTAT_HONGKONG_DAILY_RETURNS = C:/Users/Haksun Li/Documents/Businesses/Numerical Method/data/raw/compustat/hongkong/dailyReturn.1995.2011.csv.zip
+DATA_COMPUSTAT = ${DATA_FOLDER}/raw/compustat
+COMPUSTAT_HONGKONG_COMPANY = ${DATA_COMPUSTAT}/hongkong/company.csv.zip
+COMPUSTAT_HONGKONG_ACCOUNTING = ${DATA_COMPUSTAT}/hongkong/monthlyAccounting.1995.2010.csv.zip
+COMPUSTAT_HONGKONG_DAILY_RETURNS = ${DATA_COMPUSTAT}/hongkong/dailyReturn.1995.2011.csv.zip
 
 #This folder contains the data downloaded from the HKMA website.
-HKMA = C:/Users/Haksun Li/Documents/Businesses/Numerical Method/data/raw/HKMA
+HKMA = ${DATA_FOLDER}/raw/HKMA
 

core/src/test/java/com/numericalmethod/algoquant/config/AlgoQuantConfigTest.java

@@ -23 +23 @@
 package com.numericalmethod.algoquant.config;
 
+import org.junit.AfterClass;
 import static org.junit.Assert.*;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
@@ -31 +33 @@
  */
 public class AlgoQuantConfigTest {
+    
+    public static String currentConfigFile;
+    
+    @BeforeClass
+    public static void saveConfigFile() {
+        currentConfigFile = AlgoQuantConfig.getConfiguration().getConfigFilePath();
+    }
+    
+    @AfterClass
+    public static void restoreConfigFile() {
+        AlgoQuantConfig.getConfiguration().setConfigFile(currentConfigFile);
+    }
+    
 
     @Test
@@ -36 +51 @@
         AlgoQuantConfig instance = AlgoQuantConfig.getConfiguration();
         instance.setConfigFile("config.default.xml");
-        String HOLIDAY_DIR = instance.getString(ConfigProperties.HOLIDAY_DIR.getPropertyName());
-        assertEquals("./resources/holidays", HOLIDAY_DIR);
+        String HOLIDAY_DIR = instance.getString(ConfigProperties.DATA_HOLIDAYS.getPropertyName());
+        assertEquals("./log/holidays", HOLIDAY_DIR);
     }
 
@@ -43 +58 @@
     public void testChangeConfigFile() {
         AlgoQuantConfig instance = AlgoQuantConfig.getConfiguration();
-        assertEquals("./resources/holidays", instance.getString(ConfigProperties.HOLIDAY_DIR.getPropertyName()));
+        assertEquals("./log/holidays", instance.getString(ConfigProperties.DATA_HOLIDAYS.getPropertyName()));
 
         instance.setConfigFile("./test-data/config/test.config.xml");
         AlgoQuantConfig instance2 = AlgoQuantConfig.getConfiguration();
-        assertEquals("./test-data", instance2.getString(ConfigProperties.HOLIDAY_DIR.getPropertyName()));
+        assertEquals("./test-data", instance2.getString(ConfigProperties.DATA_HOLIDAYS.getPropertyName()));
 
         assertEquals(instance, instance2);
@@ -56 +71 @@
         AlgoQuantConfig instance = AlgoQuantConfig.getConfiguration();
         instance.setConfigFile("config.default.xml");
-        assertEquals("./resources/holidays", instance.getString(ConfigProperties.HOLIDAY_DIR.getPropertyName()));
+        assertEquals("./log/holidays", instance.getString(ConfigProperties.DATA_HOLIDAYS.getPropertyName()));
 
         instance.setString("key", "value");

core/src/test/java/com/numericalmethod/algoquant/data/calendar/holiday/HolidaysFromFileTest.java

@@ -24 +24 @@
 
 import com.numericalmethod.algoquant.config.AlgoQuantConfig;
+import com.numericalmethod.algoquant.config.ConfigProperties;
 import com.numericalmethod.algoquant.data.calendar.TimeZoneUtils;
 import static com.numericalmethod.nmutils.NMUtils.*;
@@ -50 +51 @@
     public static void setUpClass() throws Exception {
         AlgoQuantConfig instance = AlgoQuantConfig.getConfiguration();
-        instance.setString("HOLIDAY_DIR", HOLIDAY_DIR);
-        NMIO.newFolder(HOLIDAY_DIR);
+        instance.setString(ConfigProperties.DATA_HOLIDAYS.getPropertyName(), HOLIDAY_DIR);
     }
 
     @Test
     public void testDateOutOfRange() throws IOException {
-
         //US
         DateTime queryDate = JodaTimeUtils.getDate(1929, 12, 31, ExchangeIndex.NYSE.getTimeZone());
@@ -75 +74 @@
     @Test
     public void testBuildHolidayFile() {
-        clearHolidayDir();
+        NMIO.deleteDir(new File(HOLIDAY_DIR));
 
         //US
@@ -96 +95 @@
     @Test
     public void testUpdateExistingHolidayFile() throws IOException {
-        clearHolidayDir();
+        NMIO.deleteDir(new File(HOLIDAY_DIR));
 
         //US
@@ -140 +139 @@
     @Test
     public void testHolidayFileExistandUpdate() {
-
         //US
         DateTime queryDate = JodaTimeUtils.getDate(2011, 11, 24, ExchangeIndex.NYSE.getTimeZone());
@@ -166 +164 @@
     }
 
-//    @Test
+//    @Test // TODO: fix it
     public void testOffline() {
-        //Disable network interface before running this test
-        clearHolidayDir();
-
-        //US
-        DateTime queryDate = JodaTimeUtils.getDate(2011, 11, 24, ExchangeIndex.NYSE.getTimeZone());
-        HolidaysFromFile holidayFile = new HolidaysFromFile(ExchangeIndex.NYSE);
+        // remove all local data files first
+        NMIO.deleteDir(new File(HOLIDAY_DIR)); // TODO: sometimes fails
+
+        //US
+        DateTime queryDate = JodaTimeUtils.getDate(2011, 11, 24, ExchangeIndex.NYSE.getTimeZone());
+        HolidaysFromFile holidayFile = new HolidaysFromFile(ExchangeIndex.NYSE, false);
         assertFalse(holidayFile.isHoliday(queryDate));
 
@@ -181 +179 @@
         //HK
         queryDate = JodaTimeUtils.getDate(2004, 4, 9, ExchangeIndex.HKEX.getTimeZone());
-        holidayFile = new HolidaysFromFile(ExchangeIndex.HKEX);
+        holidayFile = new HolidaysFromFile(ExchangeIndex.HKEX, false);
         assertFalse(holidayFile.isHoliday(queryDate));
 
@@ -189 +187 @@
         //SG
         queryDate = JodaTimeUtils.getDate(2005, 3, 25, ExchangeIndex.SGX.getTimeZone());
-        holidayFile = new HolidaysFromFile(ExchangeIndex.SGX);
+        holidayFile = new HolidaysFromFile(ExchangeIndex.SGX, false);
         assertFalse(holidayFile.isHoliday(queryDate));
 
         queryDate = JodaTimeUtils.getDate(2007, 8, 9, ExchangeIndex.SGX.getTimeZone());
         assertFalse(holidayFile.isHoliday(queryDate));
-    }
-
-    private void clearHolidayDir() {
-        File directory = new File(HOLIDAY_DIR);
-
-        File[] files = directory.listFiles();
-
-        for (File file : files) {
-            file.delete();
-        }
     }
 

core/test-data/config/data.properties

@@ -1 @@
-HOLIDAY_DIR = ./test-data
+DATA_HOLIDAYS = ./test-data
 ABC_DIR = ./abc

core/test-data/config/data2.properties

@@ -1 +1 @@
 #the folder that contains the Numerical Method Inc. historical data
-HOLIDAY_DIR = ./resources/holidays
+DATA_HOLIDAYS = ./log/holidays
 ABC_DIR = ./bbc
 TEST_DIR = ./hhh