WarcWriter

• WARC/1.0 요구사항에 따라 crawl 데이터들을 warc(Web ARChive) 파일로 저장한다.
  warc 포맷은 InternetArchive's ARC File Format의 개정 포맷이며 전통적으로 web crawl을 저장하기 위해 사용된다.

기존 설정 값

<bean id="warcWriter" class="org.archive.modules.writer.WARCWriterProcessor">
  <!-- <property name="compress" value="true" /> -->
  <!-- <property name="prefix" value="WEB" /> -->
  <!-- <property name="suffix" value="${HOSTNAME}" /> -->
  <!-- <property name="maxFileSizeBytes" value="1000000000" /> -->
  <!-- <property name="poolMaxActive" value="1" /> -->
  <!-- <property name="MaxWaitForIdleMs" value="500" /> -->
  <!-- <property name="skipIdenticalDigests" value="false" /> -->
  <!-- <property name="maxTotalBytesToWrite" value="0" /> -->
  <!-- <property name="directory" value="${launchId}" /> -->
  <!-- <property name="storePaths">
        <list>
         <value>warcs</value>
        </list>
       </property> -->
  <!-- <property name="template" value="${prefix}-${timestamp17}-${serialno}-${heritrix.pid}~${heritrix.hostname}~${heritrix.port}" /> -->
  <!-- <property name="writeRequests" value="true" /> -->
  <!-- <property name="writeMetadata" value="true" /> -->
  <!-- <property name="writeRevisitForIdenticalDigests" value="true" /> -->
  <!-- <property name="writeRevisitForNotModified" value="true" /> -->
  <!-- <property name="startNewFilesOnCheckpoint" value="true" /> -->