企业搜索引擎开发之连接器connector（十）

这里分析一下FeedConnection接口及其实现类GsaFeedConnection相关源码：

FeedConnection接口源码如下： 

/**
 * Interface for a feed connection. This takes in a data source name and a data
 * source object that contains the data to be sent.  The actual connection to
 * the feed server should be established by the implementation during
 * construction or initialization.
 */
public interface FeedConnection {

  /**
   * Sends data contained in the given data object identified as the given data
   * source name.
   *
   * @param feedData an object that encapsulates the feed data that needs to be
   *        sent by the <code>FeedConnection</code>.
   * @return response from the feed server.
   * @throws FeedException if problem extracting the data or sending it.
   * @throws RepositoryException if problem retrieving data from the Connector.
   */
  public String sendData(FeedData feedData)
      throws FeedException, RepositoryException;

  /**
   * Returns true if the Feed host has large number of unprocessed Feed items.
   * The Feed host may temporarily stop processing Feed items during periodic
   * maintenance, when resetting the index, during system configuration, or
   * due to certain error conditions. If backlogged, the Feed client may choose
   * to throttle back its feeds until the backlog clears.
   *
   * @return true if the Feed host is known to be backlogged processing feeds,
   *         false otherwise.
   */
  public boolean isBacklogged();

  /**
   * Return a String consisting of a comma-separated list supported content
   * encodings.  For instance: "base64binary, base64compressed".
   *
   * @return supported content encodings.
   */
  public String getContentEncodings();

}

方法String sendData(FeedData feedData)向应用中心发送数据；boolean isBacklogged()检查应用中心是否可用；String getContentEncodings()检测应用中心支持的编码类型

实现类GsaFeedConnection源码如下：

/**
 * Opens a connection to a url and sends data to it.
 */
public class GsaFeedConnection implements FeedConnection {

  /**
   * The GSA's response when it successfully receives a feed.
   */
  public static final String SUCCESS_RESPONSE = "Success";

  /**
   * The GSA's response when the client is not authorized to send feeds.
   */
  public static final String UNAUTHORIZED_RESPONSE =
      "Error - Unauthorized Request";

  /**
   * The GSA's response when it runs out of disk space.
   */
  public static final String DISKFULL_RESPONSE =
      "Feed not accepted due to insufficient disk space.";

  /**
   * The GSA's response when there was an internal error.
   */
  public static final String INTERNAL_ERROR_RESPONSE = "Internal Error";

  // Multipart/form-data uploads require a boundary to delimit controls.
  // Since we XML-escape or base64-encode all data provided by the connector,
  // the feed XML will never contain "<<".
  private static final String BOUNDARY = "<<";

  private static final String CRLF = "\r\n";

  // Content encodings supported by GSA.
  private String contentEncodings = null;

  // True if we recently got a feed error of some sort.
  private boolean gotFeedError = false;

  // XmlFeed URL
  private URL feedUrl = null;

  // XmlFeed DTD URL
  private URL dtdUrl = null;

  // BacklogCount URL
  private URL backlogUrl = null;

  // BacklogCount Ceiling. Throttle back feed if backlog exceeds the ceiling.
  private int backlogCeiling = 50000;

  // BacklogCount Floor. Stop throttling feed if backlog drops below floor.
  private int backlogFloor = 15000;

  // True if the feed is throttled back due to excessive backlog.
  private boolean isBacklogged = false;

  // Time of last backlog check.
  private long lastBacklogCheck;

  // How often to check for backlog (in milliseconds).
  private long backlogCheckInterval = 15 * 60 * 1000L;

  private static final Logger LOGGER =
      Logger.getLogger(GsaFeedConnection.class.getName());

  public GsaFeedConnection(String host, int port) throws MalformedURLException {
    this.setFeedHostAndPort(host, port);
  }

  public synchronized void setFeedHostAndPort(String host, int port)
      throws MalformedURLException {
    feedUrl = new URL("http", host, port, "/xmlfeed");
    dtdUrl = new URL("http", host, port, "/getdtd");
    contentEncodings = null;
    backlogUrl = new URL("http", host, port, "/getbacklogcount");
    lastBacklogCheck = 0L;
  }

  /**
   * Set the backlog check parameters. The Feed connection can check to see
   * if the GSA is falling behind processing feeds by calling the GSA's
   * {@code getbacklogcount} servlet. If the number of outstanding feed
   * items exceeds the {@code ceiling}, then the GSA is considered
   * backlogged.  If the number of outstanding feed items then drops below
   * the {@code floor}, it may be considered no longer backlogged.
   *
   * @param floor backlog count floor value, below which the GSA is no
   *        longer considered backlogged.
   * @param ceiling backlog count ceiling value, above which the GSA is
   *        considered backlogged.
   * @param interval number of seconds to wait between backlog count checks.
   */
  public void setBacklogCheck(int floor, int ceiling, int interval) {
    backlogFloor = floor;
    backlogCeiling = ceiling;
    backlogCheckInterval = interval * 1000L;
  }

  public void setContentEncodings(String contentEncodings) {
    this.contentEncodings = contentEncodings;
  }

  private static final void controlHeader(StringBuilder builder,
        String name, String mimetype) {
    builder.append("--").append(BOUNDARY).append(CRLF);
    builder.append("Content-Disposition: form-data;");
    builder.append(" name=\"").append(name).append("\"").append(CRLF);
    builder.append("Content-Type: ").append(mimetype).append(CRLF);
    builder.append(CRLF);
  }

  /* @Override */
  public String sendData(FeedData feedData)
      throws FeedException {
    try {
      String response = sendFeedData((XmlFeed)feedData);
      gotFeedError = !response.equalsIgnoreCase(SUCCESS_RESPONSE);
      return response;
    } catch (FeedException fe) {
      gotFeedError = true;
      throw fe;
    }
  }

  private String sendFeedData(XmlFeed feed)
      throws FeedException {
    String feedType = feed.getFeedType();
    String dataSource = feed.getDataSource();
    OutputStream outputStream;
    HttpURLConnection uc;
    StringBuilder buf = new StringBuilder();
    byte[] prefix;
    byte[] suffix;
    try {
      // Build prefix.
      controlHeader(buf, "datasource", ServletUtil.MIMETYPE_TEXT_PLAIN);
      buf.append(dataSource).append(CRLF);
      controlHeader(buf, "feedtype", ServletUtil.MIMETYPE_TEXT_PLAIN);
      buf.append(feedType).append(CRLF);
      controlHeader(buf, "data", ServletUtil.MIMETYPE_XML);
      prefix = buf.toString().getBytes("UTF-8");

      // Build suffix.
      buf.setLength(0);
      buf.append(CRLF).append("--").append(BOUNDARY).append("--").append(CRLF);
      suffix = buf.toString().getBytes("UTF-8");

      LOGGER.finest("Opening feed connection.");
      synchronized (this) {
        uc = (HttpURLConnection) feedUrl.openConnection();
      }
      uc.setDoInput(true);
      uc.setDoOutput(true);
      uc.setFixedLengthStreamingMode(prefix.length + feed.size()
          + suffix.length);
      uc.setRequestProperty("Content-Type", "multipart/form-data; boundary="
          + BOUNDARY);
      outputStream = uc.getOutputStream();
    } catch (IOException ioe) {
      throw new FeedException(ioe);
    }

    boolean isThrowing = false;
    buf.setLength(0);
    try {
      LOGGER.finest("Writing feed data to feed connection.");
      // If there is an exception during this read/write, we do our
      // best to close the url connection and read the result.
      try {
        outputStream.write(prefix);
        feed.writeTo(outputStream);
        outputStream.write(suffix);
        outputStream.flush();
      } catch (IOException e) {
        LOGGER.log(Level.SEVERE,
            "IOException while posting: will retry later", e);
        isThrowing = true;
        throw new FeedException(e);
      } catch (RuntimeException e) {
        isThrowing = true;
        throw e;
      } catch (Error e) {
        isThrowing = true;
        throw e;
      } finally {
        try {
          outputStream.close();
        } catch (IOException e) {
          LOGGER.log(Level.SEVERE,
              "IOException while closing after post: will retry later", e);
          if (!isThrowing) {
            isThrowing = true;
            throw new FeedException(e);
          }
        }
      }
    } finally {
      BufferedReader br = null;
      try {
        LOGGER.finest("Waiting for response from feed connection.");
        InputStream inputStream = uc.getInputStream();
        br = new BufferedReader(new InputStreamReader(inputStream, "UTF8"));
        String line;
        while ((line = br.readLine()) != null) {
          buf.append(line);
        }
      } catch (IOException ioe) {
        if (!isThrowing) {
          throw new FeedException(ioe);
        }
      } finally {
        try {
          if (br != null) {
            br.close();
          }
        } catch (IOException e) {
          LOGGER.log(Level.SEVERE,
                     "IOException while closing after post: continuing", e);
        }
        if (uc != null) {
          uc.disconnect();
        }
        if (LOGGER.isLoggable(Level.FINEST)) {
          LOGGER.finest("Received response from feed connection: "
                        + buf.toString());
        }
      }
    }
    return buf.toString();
  }

  /* @Override */
  public synchronized String getContentEncodings() {
    if (contentEncodings == null) {
      String dtd = getDtd();
      if (dtd == null) {
        // Failed to get a DTD. Assume the GSA only supports base64 encoded.
        contentEncodings = "base64binary";
      } else {
        // TODO: Extract the supported content encodings from the DTD.
        // As of GSA 6.2, returning a DTD at all also means compression
        // is supported.
        contentEncodings = "base64binary,base64compressed";
      }
      if (LOGGER.isLoggable(Level.FINE)) {
        LOGGER.fine("GSA supports Content Encodings: " + contentEncodings);
      }
    }
    return contentEncodings;
  }

  /* @Override */
  public synchronized boolean isBacklogged() {
    if (lastBacklogCheck != Long.MAX_VALUE) {
      long now = System.currentTimeMillis();
      if ((now - lastBacklogCheck) > backlogCheckInterval) {
        lastBacklogCheck = now;
        // If we got a feed error and the feed is still down, delay.
        if (gotFeedError) {
          if (isFeedAvailable()) {
            gotFeedError = false;
          } else {
            // Feed is still unavailable.
            return true;
          }
        }
        try {
          int backlogCount = getBacklogCount();
          if (backlogCount >= 0) {
            if (isBacklogged) {
              // If we were backlogged, but have dropped below the
              // floor value, then we are no longer backlogged.
              if (backlogCount < backlogFloor) {
                isBacklogged = false;
                LOGGER.info("Resuming traversal after feed backlog clears.");
              }
            } else if (backlogCount > backlogCeiling) {
              // If the backlogcount exceeds the ceiling value,
              // then we are definitely backlogged.
              isBacklogged = true;
              LOGGER.info("Pausing traversal due to excessive feed backlog.");
            }
          }
        } catch (UnsupportedOperationException e) {
          // This older GSA does not support getbacklogcount.
          // Assume never backlogged and don't check again.
          isBacklogged = false;
          lastBacklogCheck = Long.MAX_VALUE;
          LOGGER.fine("Older GSA lacks backlogcount support.");
        }
      }
    }
    return isBacklogged;
  }

  /**
   * @return the current feed backlog count of the GSA,
   *         or -1 if the count is unavailable.
   * @throws UnsupportedOperationException if the GSA does
   *         not support getbacklogcount.
   */
  private int getBacklogCount() {
    try {
      HttpResponse response = doGet(backlogUrl, "backlogcount");
      if (response != null && response.content != null) {
        return Integer.parseInt(response.content);
      }
    } catch (NumberFormatException ignored) {
      // Got a non-integer backlog count - probably an error message,
      // which we have already logged (at Finest).  Simply return -1,
      // indicating that the backlogcount is not currently available.
    }
    // If we get here something bad happened.  It is not the case that the
    // GSA doesn't support getbacklogcount, but we still failed to retrieve it.
    return -1;
  }

  /**
   * Tests for feed error conditions such as insufficient disk space,
   * unauthorized clients, etc.  If the /xmlfeed command is sent with no
   * arguments, the server will return an error message and a 200 response
   * code if it can't accept feeds.  If it can continue to accept feeds, then
   * it will return a 400 bad request since it's missing required parameters.
   *
   * @return True if feed host is likely to accept a feed request.
   */
  private boolean isFeedAvailable() {
    try {
      HttpResponse response = doGet(feedUrl, "XmlFeed");
      if (response != null) {
        if (response.responseCode == HttpURLConnection.HTTP_BAD_REQUEST) {
          // The expected responseCode if no error conditions are present.
          LOGGER.finest("XmlFeed connection seems to be accepting new feeds.");
          return true;
        }
        if (response.content != null) {
          response.content.contains(SUCCESS_RESPONSE);
        }
      }
    } catch (UnsupportedOperationException ignored) {
      // This GSA does not support feeds?  Return false.
    }
    // If we get here something bad happened.
    return false;
  }

  /**
   * @return the current feed XML DTD for the GSA,
   *         or null if the DTD is unavailable.
   */
  private String getDtd() {
    try {
      HttpResponse response = doGet(dtdUrl, "DTD");
      if (response != null && response.content != null) {
        return response.content;
      }
    } catch (UnsupportedOperationException ignored) {
      // This older GSA does not support getdtd, so return null.
      LOGGER.fine("Older GSA lacks get DTD support.");
    }
    return null;
  }

  /**
   * Get the response to a URL request.  The response is returned
   * as an HttpResponse containing the HTTP ResponseCode and the
   * returned content as a String. The content String is only returned
   * if the response code was OK.
   *
   * @param url the URL to request
   * @param name the name of the feature requested (for logging)
   * @return HttpResponse representing response to an HTTP GET.
   *         or null if the GSA is unavailable.
   * @throws UnsupportedOperationException if the GSA does
   *         not support the requested feature.
   */
  private HttpResponse doGet(URL url, String name) {
    HttpURLConnection conn = null;
    BufferedReader br = null;
    String str = null;
    StringBuilder buf = new StringBuilder();
    try {
      if (LOGGER.isLoggable(Level.FINEST)) {
        LOGGER.finest("Opening " + name + " connection.");
      }
      conn = (HttpURLConnection)url.openConnection();
      conn.connect();
      int responseCode = conn.getResponseCode();
      if (responseCode == HttpURLConnection.HTTP_OK) {
        br = new BufferedReader(new InputStreamReader(conn.getInputStream(),
                                                      "UTF8"));
        while ((str = br.readLine()) != null) {
          buf.append(str);
        }
        str = buf.toString().trim();
        if (LOGGER.isLoggable(Level.FINEST)) {
          LOGGER.finest("Received " + name + ": " + str);
        }
        return new HttpResponse(responseCode, str);
      } else if (responseCode == HttpURLConnection.HTTP_NOT_FOUND) {
        throw new UnsupportedOperationException(
            "GSA lacks " + name + " support.");
      } else {
        return new HttpResponse(responseCode);
      }
    } catch (IOException ioe) {
      LOGGER.finest("Error while reading " + name + ": " + ioe.getMessage());
    } finally {
      try {
        if (br != null) {
          br.close();
        }
      } catch (IOException e) {
        LOGGER.finest("Error after reading " + name + ": " + e.getMessage());
      }
      if (conn != null) {
        conn.disconnect();
      }
    }
    // If we get here something bad happened. It is not the case that the GSA
    // doesn't support the requested feature, but we failed to retrieve it.
    return null;
  }

  private static class HttpResponse {
    public int responseCode;  // The HTTP response code.
    public String content;    // The returned content as a String.

    public HttpResponse(int responseCode) {
      this(responseCode, null);
    }

    public HttpResponse(int responseCode, String content) {
      this.responseCode = responseCode;
      this.content = content;
    }
  }
}

在String sendFeedData(XmlFeed feed)方法中是通过HttpURLConnection类来发送数据的，向HttpURLConnection的OutputStream输出流写入xmlfeed数据

---------------------------------------------------------------------------

本系列企业搜索引擎开发之连接器connector系本人原创

转载请注明出处 博客园 刺猬的温驯

本文链接http://www.cnblogs.com/chenying99/archive/2013/03/19/2968427.html