1124 lines
36 KiB
Java
1124 lines
36 KiB
Java
/*
|
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
|
* contributor license agreements. See the NOTICE file distributed with
|
|
* this work for additional information regarding copyright ownership.
|
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
|
* (the "License"); you may not use this file except in compliance with
|
|
* the License. You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
package org.apache.commons.text;
|
|
|
|
import java.util.ArrayList;
|
|
import java.util.Collections;
|
|
import java.util.List;
|
|
import java.util.ListIterator;
|
|
import java.util.NoSuchElementException;
|
|
|
|
import org.apache.commons.lang3.ArrayUtils;
|
|
import org.apache.commons.lang3.StringUtils;
|
|
|
|
/**
|
|
* Tokenizes a string based on delimiters (separators)
|
|
* and supporting quoting and ignored character concepts.
|
|
* <p>
|
|
* This class can split a String into many smaller strings. It aims
|
|
* to do a similar job to {@link java.util.StringTokenizer StringTokenizer},
|
|
* however it offers much more control and flexibility including implementing
|
|
* the {@code ListIterator} interface. By default, it is set up
|
|
* like {@code StringTokenizer}.
|
|
* <p>
|
|
* The input String is split into a number of <i>tokens</i>.
|
|
* Each token is separated from the next String by a <i>delimiter</i>.
|
|
* One or more delimiter characters must be specified.
|
|
* <p>
|
|
* Each token may be surrounded by quotes.
|
|
* The <i>quote</i> matcher specifies the quote character(s).
|
|
* A quote may be escaped within a quoted section by duplicating itself.
|
|
* <p>
|
|
* Between each token and the delimiter are potentially characters that need trimming.
|
|
* The <i>trimmer</i> matcher specifies these characters.
|
|
* One usage might be to trim whitespace characters.
|
|
* <p>
|
|
* At any point outside the quotes there might potentially be invalid characters.
|
|
* The <i>ignored</i> matcher specifies these characters to be removed.
|
|
* One usage might be to remove new line characters.
|
|
* <p>
|
|
* Empty tokens may be removed or returned as null.
|
|
* <pre>
|
|
* "a,b,c" - Three tokens "a","b","c" (comma delimiter)
|
|
* " a, b , c " - Three tokens "a","b","c" (default CSV processing trims whitespace)
|
|
* "a, ", b ,", c" - Three tokens "a, " , " b ", ", c" (quoted text untouched)
|
|
* </pre>
|
|
*
|
|
* <table>
|
|
* <caption>StrTokenizer properties and options</caption>
|
|
* <tr>
|
|
* <th>Property</th><th>Type</th><th>Default</th>
|
|
* </tr>
|
|
* <tr>
|
|
* <td>delim</td><td>CharSetMatcher</td><td>{ \t\n\r\f}</td>
|
|
* </tr>
|
|
* <tr>
|
|
* <td>quote</td><td>NoneMatcher</td><td>{}</td>
|
|
* </tr>
|
|
* <tr>
|
|
* <td>ignore</td><td>NoneMatcher</td><td>{}</td>
|
|
* </tr>
|
|
* <tr>
|
|
* <td>emptyTokenAsNull</td><td>boolean</td><td>false</td>
|
|
* </tr>
|
|
* <tr>
|
|
* <td>ignoreEmptyTokens</td><td>boolean</td><td>true</td>
|
|
* </tr>
|
|
* </table>
|
|
*
|
|
* @since 1.0
|
|
* @deprecated Deprecated as of 1.3, use {@link StringTokenizer} instead. This class will be removed in 2.0.
|
|
*/
|
|
@Deprecated
|
|
@SuppressWarnings("index") // class is deprecated
|
|
public class StrTokenizer implements ListIterator<String>, Cloneable {
|
|
|
|
/** Comma separated values tokenizer internal variable. */
|
|
private static final StrTokenizer CSV_TOKENIZER_PROTOTYPE;
|
|
|
|
/** Tab separated values tokenizer internal variable. */
|
|
private static final StrTokenizer TSV_TOKENIZER_PROTOTYPE;
|
|
|
|
static {
|
|
CSV_TOKENIZER_PROTOTYPE = new StrTokenizer();
|
|
CSV_TOKENIZER_PROTOTYPE.setDelimiterMatcher(StrMatcher.commaMatcher());
|
|
CSV_TOKENIZER_PROTOTYPE.setQuoteMatcher(StrMatcher.doubleQuoteMatcher());
|
|
CSV_TOKENIZER_PROTOTYPE.setIgnoredMatcher(StrMatcher.noneMatcher());
|
|
CSV_TOKENIZER_PROTOTYPE.setTrimmerMatcher(StrMatcher.trimMatcher());
|
|
CSV_TOKENIZER_PROTOTYPE.setEmptyTokenAsNull(false);
|
|
CSV_TOKENIZER_PROTOTYPE.setIgnoreEmptyTokens(false);
|
|
|
|
TSV_TOKENIZER_PROTOTYPE = new StrTokenizer();
|
|
TSV_TOKENIZER_PROTOTYPE.setDelimiterMatcher(StrMatcher.tabMatcher());
|
|
TSV_TOKENIZER_PROTOTYPE.setQuoteMatcher(StrMatcher.doubleQuoteMatcher());
|
|
TSV_TOKENIZER_PROTOTYPE.setIgnoredMatcher(StrMatcher.noneMatcher());
|
|
TSV_TOKENIZER_PROTOTYPE.setTrimmerMatcher(StrMatcher.trimMatcher());
|
|
TSV_TOKENIZER_PROTOTYPE.setEmptyTokenAsNull(false);
|
|
TSV_TOKENIZER_PROTOTYPE.setIgnoreEmptyTokens(false);
|
|
}
|
|
|
|
/**
|
|
* Returns a clone of {@code CSV_TOKENIZER_PROTOTYPE}.
|
|
*
|
|
* @return a clone of {@code CSV_TOKENIZER_PROTOTYPE}.
|
|
*/
|
|
private static StrTokenizer getCSVClone() {
|
|
return (StrTokenizer) CSV_TOKENIZER_PROTOTYPE.clone();
|
|
}
|
|
|
|
/**
|
|
* Gets a new tokenizer instance which parses Comma Separated Value strings
|
|
* initializing it with the given input. The default for CSV processing
|
|
* will be trim whitespace from both ends (which can be overridden with
|
|
* the setTrimmer method).
|
|
* <p>
|
|
* You must call a "reset" method to set the string which you want to parse.
|
|
* </p>
|
|
* @return a new tokenizer instance which parses Comma Separated Value strings
|
|
*/
|
|
public static StrTokenizer getCSVInstance() {
|
|
return getCSVClone();
|
|
}
|
|
|
|
/**
|
|
* Gets a new tokenizer instance which parses Comma Separated Value strings
|
|
* initializing it with the given input. The default for CSV processing
|
|
* will be trim whitespace from both ends (which can be overridden with
|
|
* the setTrimmer method).
|
|
*
|
|
* @param input the text to parse
|
|
* @return a new tokenizer instance which parses Comma Separated Value strings
|
|
*/
|
|
public static StrTokenizer getCSVInstance(final char[] input) {
|
|
final StrTokenizer tok = getCSVClone();
|
|
tok.reset(input);
|
|
return tok;
|
|
}
|
|
|
|
/**
|
|
* Gets a new tokenizer instance which parses Comma Separated Value strings
|
|
* initializing it with the given input. The default for CSV processing
|
|
* will be trim whitespace from both ends (which can be overridden with
|
|
* the setTrimmer method).
|
|
*
|
|
* @param input the text to parse
|
|
* @return a new tokenizer instance which parses Comma Separated Value strings
|
|
*/
|
|
public static StrTokenizer getCSVInstance(final String input) {
|
|
final StrTokenizer tok = getCSVClone();
|
|
tok.reset(input);
|
|
return tok;
|
|
}
|
|
/**
|
|
* Returns a clone of {@code TSV_TOKENIZER_PROTOTYPE}.
|
|
*
|
|
* @return a clone of {@code TSV_TOKENIZER_PROTOTYPE}.
|
|
*/
|
|
private static StrTokenizer getTSVClone() {
|
|
return (StrTokenizer) TSV_TOKENIZER_PROTOTYPE.clone();
|
|
}
|
|
|
|
/**
|
|
* Gets a new tokenizer instance which parses Tab Separated Value strings.
|
|
* The default for CSV processing will be trim whitespace from both ends
|
|
* (which can be overridden with the setTrimmer method).
|
|
* <p>
|
|
* You must call a "reset" method to set the string which you want to parse.
|
|
* </p>
|
|
* @return a new tokenizer instance which parses Tab Separated Value strings.
|
|
*/
|
|
public static StrTokenizer getTSVInstance() {
|
|
return getTSVClone();
|
|
}
|
|
|
|
/**
|
|
* Gets a new tokenizer instance which parses Tab Separated Value strings.
|
|
* The default for CSV processing will be trim whitespace from both ends
|
|
* (which can be overridden with the setTrimmer method).
|
|
* @param input the string to parse
|
|
* @return a new tokenizer instance which parses Tab Separated Value strings.
|
|
*/
|
|
public static StrTokenizer getTSVInstance(final char[] input) {
|
|
final StrTokenizer tok = getTSVClone();
|
|
tok.reset(input);
|
|
return tok;
|
|
}
|
|
|
|
/**
|
|
* Gets a new tokenizer instance which parses Tab Separated Value strings.
|
|
* The default for CSV processing will be trim whitespace from both ends
|
|
* (which can be overridden with the setTrimmer method).
|
|
* @param input the string to parse
|
|
* @return a new tokenizer instance which parses Tab Separated Value strings.
|
|
*/
|
|
public static StrTokenizer getTSVInstance(final String input) {
|
|
final StrTokenizer tok = getTSVClone();
|
|
tok.reset(input);
|
|
return tok;
|
|
}
|
|
|
|
/** The text to work on. */
|
|
private char[] chars;
|
|
|
|
/** The parsed tokens. */
|
|
private String[] tokens;
|
|
|
|
/** The current iteration position. */
|
|
private int tokenPos;
|
|
|
|
/** The delimiter matcher. */
|
|
private StrMatcher delimMatcher = StrMatcher.splitMatcher();
|
|
|
|
/** The quote matcher. */
|
|
private StrMatcher quoteMatcher = StrMatcher.noneMatcher();
|
|
|
|
/** The ignored matcher. */
|
|
private StrMatcher ignoredMatcher = StrMatcher.noneMatcher();
|
|
|
|
/** The trimmer matcher. */
|
|
private StrMatcher trimmerMatcher = StrMatcher.noneMatcher();
|
|
|
|
/** Whether to return empty tokens as null. */
|
|
private boolean emptyAsNull;
|
|
|
|
/** Whether to ignore empty tokens. */
|
|
private boolean ignoreEmptyTokens = true;
|
|
|
|
/**
|
|
* Constructs a tokenizer splitting on space, tab, newline and form feed
|
|
* as per StringTokenizer, but with no text to tokenize.
|
|
* <p>
|
|
* This constructor is normally used with {@link #reset(String)}.
|
|
* </p>
|
|
*/
|
|
public StrTokenizer() {
|
|
this.chars = null;
|
|
}
|
|
|
|
/**
|
|
* Constructs a tokenizer splitting on space, tab, newline and form feed
|
|
* as per StringTokenizer.
|
|
*
|
|
* @param input the string which is to be parsed, not cloned
|
|
*/
|
|
public StrTokenizer(final char[] input) {
|
|
if (input == null) {
|
|
this.chars = null;
|
|
} else {
|
|
this.chars = input.clone();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Constructs a tokenizer splitting on the specified character.
|
|
*
|
|
* @param input the string which is to be parsed, not cloned
|
|
* @param delim the field delimiter character
|
|
*/
|
|
public StrTokenizer(final char[] input, final char delim) {
|
|
this(input);
|
|
setDelimiterChar(delim);
|
|
}
|
|
|
|
/**
|
|
* Constructs a tokenizer splitting on the specified delimiter character
|
|
* and handling quotes using the specified quote character.
|
|
*
|
|
* @param input the string which is to be parsed, not cloned
|
|
* @param delim the field delimiter character
|
|
* @param quote the field quoted string character
|
|
*/
|
|
public StrTokenizer(final char[] input, final char delim, final char quote) {
|
|
this(input, delim);
|
|
setQuoteChar(quote);
|
|
}
|
|
|
|
/**
|
|
* Constructs a tokenizer splitting on the specified string.
|
|
*
|
|
* @param input the string which is to be parsed, not cloned
|
|
* @param delim the field delimiter string
|
|
*/
|
|
public StrTokenizer(final char[] input, final String delim) {
|
|
this(input);
|
|
setDelimiterString(delim);
|
|
}
|
|
|
|
/**
|
|
* Constructs a tokenizer splitting using the specified delimiter matcher.
|
|
*
|
|
* @param input the string which is to be parsed, not cloned
|
|
* @param delim the field delimiter matcher
|
|
*/
|
|
public StrTokenizer(final char[] input, final StrMatcher delim) {
|
|
this(input);
|
|
setDelimiterMatcher(delim);
|
|
}
|
|
|
|
/**
|
|
* Constructs a tokenizer splitting using the specified delimiter matcher
|
|
* and handling quotes using the specified quote matcher.
|
|
*
|
|
* @param input the string which is to be parsed, not cloned
|
|
* @param delim the field delimiter character
|
|
* @param quote the field quoted string character
|
|
*/
|
|
public StrTokenizer(final char[] input, final StrMatcher delim, final StrMatcher quote) {
|
|
this(input, delim);
|
|
setQuoteMatcher(quote);
|
|
}
|
|
|
|
/**
|
|
* Constructs a tokenizer splitting on space, tab, newline and form feed
|
|
* as per StringTokenizer.
|
|
*
|
|
* @param input the string which is to be parsed
|
|
*/
|
|
public StrTokenizer(final String input) {
|
|
if (input != null) {
|
|
chars = input.toCharArray();
|
|
} else {
|
|
chars = null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Constructs a tokenizer splitting on the specified delimiter character.
|
|
*
|
|
* @param input the string which is to be parsed
|
|
* @param delim the field delimiter character
|
|
*/
|
|
public StrTokenizer(final String input, final char delim) {
|
|
this(input);
|
|
setDelimiterChar(delim);
|
|
}
|
|
|
|
/**
|
|
* Constructs a tokenizer splitting on the specified delimiter character
|
|
* and handling quotes using the specified quote character.
|
|
*
|
|
* @param input the string which is to be parsed
|
|
* @param delim the field delimiter character
|
|
* @param quote the field quoted string character
|
|
*/
|
|
public StrTokenizer(final String input, final char delim, final char quote) {
|
|
this(input, delim);
|
|
setQuoteChar(quote);
|
|
}
|
|
|
|
/**
|
|
* Constructs a tokenizer splitting on the specified delimiter string.
|
|
*
|
|
* @param input the string which is to be parsed
|
|
* @param delim the field delimiter string
|
|
*/
|
|
public StrTokenizer(final String input, final String delim) {
|
|
this(input);
|
|
setDelimiterString(delim);
|
|
}
|
|
|
|
/**
|
|
* Constructs a tokenizer splitting using the specified delimiter matcher.
|
|
*
|
|
* @param input the string which is to be parsed
|
|
* @param delim the field delimiter matcher
|
|
*/
|
|
public StrTokenizer(final String input, final StrMatcher delim) {
|
|
this(input);
|
|
setDelimiterMatcher(delim);
|
|
}
|
|
|
|
/**
|
|
* Constructs a tokenizer splitting using the specified delimiter matcher
|
|
* and handling quotes using the specified quote matcher.
|
|
*
|
|
* @param input the string which is to be parsed
|
|
* @param delim the field delimiter matcher
|
|
* @param quote the field quoted string matcher
|
|
*/
|
|
public StrTokenizer(final String input, final StrMatcher delim, final StrMatcher quote) {
|
|
this(input, delim);
|
|
setQuoteMatcher(quote);
|
|
}
|
|
|
|
/**
|
|
* Unsupported ListIterator operation.
|
|
* @param obj this parameter ignored.
|
|
* @throws UnsupportedOperationException always
|
|
*/
|
|
@Override
|
|
public void add(final String obj) {
|
|
throw new UnsupportedOperationException("add() is unsupported");
|
|
}
|
|
|
|
/**
|
|
* Adds a token to a list, paying attention to the parameters we've set.
|
|
*
|
|
* @param list the list to add to
|
|
* @param tok the token to add
|
|
*/
|
|
private void addToken(final List<String> list, String tok) {
|
|
if (tok == null || tok.isEmpty()) {
|
|
if (isIgnoreEmptyTokens()) {
|
|
return;
|
|
}
|
|
if (isEmptyTokenAsNull()) {
|
|
tok = null;
|
|
}
|
|
}
|
|
list.add(tok);
|
|
}
|
|
|
|
/**
|
|
* Checks if tokenization has been done, and if not then do it.
|
|
*/
|
|
private void checkTokenized() {
|
|
if (tokens == null) {
|
|
if (chars == null) {
|
|
// still call tokenize as subclass may do some work
|
|
final List<String> split = tokenize(null, 0, 0);
|
|
tokens = split.toArray(ArrayUtils.EMPTY_STRING_ARRAY);
|
|
} else {
|
|
final List<String> split = tokenize(chars, 0, chars.length);
|
|
tokens = split.toArray(ArrayUtils.EMPTY_STRING_ARRAY);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Creates a new instance of this Tokenizer. The new instance is reset so
|
|
* that it will be at the start of the token list.
|
|
* If a {@link CloneNotSupportedException} is caught, return {@code null}.
|
|
*
|
|
* @return a new instance of this Tokenizer which has been reset.
|
|
*/
|
|
@Override
|
|
public Object clone() {
|
|
try {
|
|
return cloneReset();
|
|
} catch (final CloneNotSupportedException ex) {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Creates a new instance of this Tokenizer. The new instance is reset so that
|
|
* it will be at the start of the token list.
|
|
*
|
|
* @return a new instance of this Tokenizer which has been reset.
|
|
* @throws CloneNotSupportedException if there is a problem cloning
|
|
*/
|
|
Object cloneReset() throws CloneNotSupportedException {
|
|
// this method exists to enable 100% test coverage
|
|
final StrTokenizer cloned = (StrTokenizer) super.clone();
|
|
if (cloned.chars != null) {
|
|
cloned.chars = cloned.chars.clone();
|
|
}
|
|
cloned.reset();
|
|
return cloned;
|
|
}
|
|
|
|
/**
|
|
* Gets the String content that the tokenizer is parsing.
|
|
*
|
|
* @return The string content being parsed
|
|
*/
|
|
public String getContent() {
|
|
if (chars == null) {
|
|
return null;
|
|
}
|
|
return new String(chars);
|
|
}
|
|
|
|
/**
|
|
* Gets the field delimiter matcher.
|
|
*
|
|
* @return The delimiter matcher in use
|
|
*/
|
|
public StrMatcher getDelimiterMatcher() {
|
|
return this.delimMatcher;
|
|
}
|
|
|
|
/**
|
|
* Gets the ignored character matcher.
|
|
* <p>
|
|
* These characters are ignored when parsing the String, unless they are
|
|
* within a quoted region.
|
|
* The default value is not to ignore anything.
|
|
* </p>
|
|
*
|
|
* @return The ignored matcher in use
|
|
*/
|
|
public StrMatcher getIgnoredMatcher() {
|
|
return ignoredMatcher;
|
|
}
|
|
|
|
/**
|
|
* Gets the quote matcher currently in use.
|
|
* <p>
|
|
* The quote character is used to wrap data between the tokens.
|
|
* This enables delimiters to be entered as data.
|
|
* The default value is '"' (double quote).
|
|
* </p>
|
|
*
|
|
* @return The quote matcher in use
|
|
*/
|
|
public StrMatcher getQuoteMatcher() {
|
|
return quoteMatcher;
|
|
}
|
|
|
|
/**
|
|
* Gets a copy of the full token list as an independent modifiable array.
|
|
*
|
|
* @return The tokens as a String array
|
|
*/
|
|
public String[] getTokenArray() {
|
|
checkTokenized();
|
|
return tokens.clone();
|
|
}
|
|
|
|
/**
|
|
* Gets a copy of the full token list as an independent modifiable list.
|
|
*
|
|
* @return The tokens as a String array
|
|
*/
|
|
public List<String> getTokenList() {
|
|
checkTokenized();
|
|
final List<String> list = new ArrayList<>(tokens.length);
|
|
Collections.addAll(list, tokens);
|
|
|
|
return list;
|
|
}
|
|
|
|
/**
|
|
* Gets the trimmer character matcher.
|
|
* <p>
|
|
* These characters are trimmed off on each side of the delimiter
|
|
* until the token or quote is found.
|
|
* The default value is not to trim anything.
|
|
* </p>
|
|
*
|
|
* @return The trimmer matcher in use
|
|
*/
|
|
public StrMatcher getTrimmerMatcher() {
|
|
return trimmerMatcher;
|
|
}
|
|
|
|
/**
|
|
* Checks whether there are any more tokens.
|
|
*
|
|
* @return true if there are more tokens
|
|
*/
|
|
@Override
|
|
public boolean hasNext() {
|
|
checkTokenized();
|
|
return tokenPos < tokens.length;
|
|
}
|
|
|
|
/**
|
|
* Checks whether there are any previous tokens that can be iterated to.
|
|
*
|
|
* @return true if there are previous tokens
|
|
*/
|
|
@Override
|
|
public boolean hasPrevious() {
|
|
checkTokenized();
|
|
return tokenPos > 0;
|
|
}
|
|
|
|
/**
|
|
* Gets whether the tokenizer currently returns empty tokens as null.
|
|
* The default for this property is false.
|
|
*
|
|
* @return true if empty tokens are returned as null
|
|
*/
|
|
public boolean isEmptyTokenAsNull() {
|
|
return this.emptyAsNull;
|
|
}
|
|
|
|
/**
|
|
* Gets whether the tokenizer currently ignores empty tokens.
|
|
* The default for this property is true.
|
|
*
|
|
* @return true if empty tokens are not returned
|
|
*/
|
|
public boolean isIgnoreEmptyTokens() {
|
|
return ignoreEmptyTokens;
|
|
}
|
|
|
|
/**
|
|
* Checks if the characters at the index specified match the quote
|
|
* already matched in readNextToken().
|
|
*
|
|
* @param srcChars the character array being tokenized
|
|
* @param pos the position to check for a quote
|
|
* @param len the length of the character array being tokenized
|
|
* @param quoteStart the start position of the matched quote, 0 if no quoting
|
|
* @param quoteLen the length of the matched quote, 0 if no quoting
|
|
* @return true if a quote is matched
|
|
*/
|
|
private boolean isQuote(final char[] srcChars,
|
|
final int pos,
|
|
final int len,
|
|
final int quoteStart,
|
|
final int quoteLen) {
|
|
for (int i = 0; i < quoteLen; i++) {
|
|
if (pos + i >= len || srcChars[pos + i] != srcChars[quoteStart + i]) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Gets the next token.
|
|
*
|
|
* @return The next String token
|
|
* @throws NoSuchElementException if there are no more elements
|
|
*/
|
|
@Override
|
|
public String next() {
|
|
if (hasNext()) {
|
|
return tokens[tokenPos++];
|
|
}
|
|
throw new NoSuchElementException();
|
|
}
|
|
|
|
/**
|
|
* Gets the index of the next token to return.
|
|
*
|
|
* @return The next token index
|
|
*/
|
|
@Override
|
|
public int nextIndex() {
|
|
return tokenPos;
|
|
}
|
|
|
|
/**
|
|
* Gets the next token from the String.
|
|
* Equivalent to {@link #next()} except it returns null rather than
|
|
* throwing {@link NoSuchElementException} when no tokens remain.
|
|
*
|
|
* @return The next sequential token, or null when no more tokens are found
|
|
*/
|
|
public String nextToken() {
|
|
if (hasNext()) {
|
|
return tokens[tokenPos++];
|
|
}
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Gets the token previous to the last returned token.
|
|
*
|
|
* @return The previous token
|
|
*/
|
|
@Override
|
|
public String previous() {
|
|
if (hasPrevious()) {
|
|
return tokens[--tokenPos];
|
|
}
|
|
throw new NoSuchElementException();
|
|
}
|
|
|
|
/**
|
|
* Gets the index of the previous token.
|
|
*
|
|
* @return The previous token index
|
|
*/
|
|
@Override
|
|
public int previousIndex() {
|
|
return tokenPos - 1;
|
|
}
|
|
|
|
/**
|
|
* Gets the previous token from the String.
|
|
*
|
|
* @return The previous sequential token, or null when no more tokens are found
|
|
*/
|
|
public String previousToken() {
|
|
if (hasPrevious()) {
|
|
return tokens[--tokenPos];
|
|
}
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Reads character by character through the String to get the next token.
|
|
*
|
|
* @param srcChars the character array being tokenized
|
|
* @param start the first character of field
|
|
* @param len the length of the character array being tokenized
|
|
* @param workArea a temporary work area
|
|
* @param tokenList the list of parsed tokens
|
|
* @return The starting position of the next field (the character
|
|
* immediately after the delimiter), or -1 if end of string found
|
|
*/
|
|
private int readNextToken(final char[] srcChars,
|
|
int start,
|
|
final int len,
|
|
final StrBuilder workArea,
|
|
final List<String> tokenList) {
|
|
// skip all leading whitespace, unless it is the
|
|
// field delimiter or the quote character
|
|
while (start < len) {
|
|
final int removeLen = Math.max(
|
|
getIgnoredMatcher().isMatch(srcChars, start, start, len),
|
|
getTrimmerMatcher().isMatch(srcChars, start, start, len));
|
|
if (removeLen == 0
|
|
|| getDelimiterMatcher().isMatch(srcChars, start, start, len) > 0
|
|
|| getQuoteMatcher().isMatch(srcChars, start, start, len) > 0) {
|
|
break;
|
|
}
|
|
start += removeLen;
|
|
}
|
|
|
|
// handle reaching end
|
|
if (start >= len) {
|
|
addToken(tokenList, StringUtils.EMPTY);
|
|
return -1;
|
|
}
|
|
|
|
// handle empty token
|
|
final int delimLen = getDelimiterMatcher().isMatch(srcChars, start, start, len);
|
|
if (delimLen > 0) {
|
|
addToken(tokenList, StringUtils.EMPTY);
|
|
return start + delimLen;
|
|
}
|
|
|
|
// handle found token
|
|
final int quoteLen = getQuoteMatcher().isMatch(srcChars, start, start, len);
|
|
if (quoteLen > 0) {
|
|
return readWithQuotes(srcChars, start + quoteLen, len, workArea, tokenList, start, quoteLen);
|
|
}
|
|
return readWithQuotes(srcChars, start, len, workArea, tokenList, 0, 0);
|
|
}
|
|
|
|
/**
|
|
* Reads a possibly quoted string token.
|
|
*
|
|
* @param srcChars the character array being tokenized
|
|
* @param start the first character of field
|
|
* @param len the length of the character array being tokenized
|
|
* @param workArea a temporary work area
|
|
* @param tokenList the list of parsed tokens
|
|
* @param quoteStart the start position of the matched quote, 0 if no quoting
|
|
* @param quoteLen the length of the matched quote, 0 if no quoting
|
|
* @return The starting position of the next field (the character
|
|
* immediately after the delimiter, or if end of string found,
|
|
* then the length of string
|
|
*/
|
|
private int readWithQuotes(final char[] srcChars, final int start, final int len, final StrBuilder workArea,
|
|
final List<String> tokenList, final int quoteStart, final int quoteLen) {
|
|
// Loop until we've found the end of the quoted
|
|
// string or the end of the input
|
|
workArea.clear();
|
|
int pos = start;
|
|
boolean quoting = quoteLen > 0;
|
|
int trimStart = 0;
|
|
|
|
while (pos < len) {
|
|
// quoting mode can occur several times throughout a string
|
|
// we must switch between quoting and non-quoting until we
|
|
// encounter a non-quoted delimiter, or end of string
|
|
if (quoting) {
|
|
// In quoting mode
|
|
|
|
// If we've found a quote character, see if it's
|
|
// followed by a second quote. If so, then we need
|
|
// to actually put the quote character into the token
|
|
// rather than end the token.
|
|
if (isQuote(srcChars, pos, len, quoteStart, quoteLen)) {
|
|
if (isQuote(srcChars, pos + quoteLen, len, quoteStart, quoteLen)) {
|
|
// matched pair of quotes, thus an escaped quote
|
|
workArea.append(srcChars, pos, quoteLen);
|
|
pos += quoteLen * 2;
|
|
trimStart = workArea.size();
|
|
continue;
|
|
}
|
|
|
|
// end of quoting
|
|
quoting = false;
|
|
pos += quoteLen;
|
|
continue;
|
|
}
|
|
|
|
} else {
|
|
// Not in quoting mode
|
|
|
|
// check for delimiter, and thus end of token
|
|
final int delimLen = getDelimiterMatcher().isMatch(srcChars, pos, start, len);
|
|
if (delimLen > 0) {
|
|
// return condition when end of token found
|
|
addToken(tokenList, workArea.substring(0, trimStart));
|
|
return pos + delimLen;
|
|
}
|
|
|
|
// check for quote, and thus back into quoting mode
|
|
if (quoteLen > 0 && isQuote(srcChars, pos, len, quoteStart, quoteLen)) {
|
|
quoting = true;
|
|
pos += quoteLen;
|
|
continue;
|
|
}
|
|
|
|
// check for ignored (outside quotes), and ignore
|
|
final int ignoredLen = getIgnoredMatcher().isMatch(srcChars, pos, start, len);
|
|
if (ignoredLen > 0) {
|
|
pos += ignoredLen;
|
|
continue;
|
|
}
|
|
|
|
// check for trimmed character
|
|
// don't yet know if its at the end, so copy to workArea
|
|
// use trimStart to keep track of trim at the end
|
|
final int trimmedLen = getTrimmerMatcher().isMatch(srcChars, pos, start, len);
|
|
if (trimmedLen > 0) {
|
|
workArea.append(srcChars, pos, trimmedLen);
|
|
pos += trimmedLen;
|
|
continue;
|
|
}
|
|
|
|
}
|
|
// copy regular character from inside quotes
|
|
workArea.append(srcChars[pos++]);
|
|
trimStart = workArea.size();
|
|
}
|
|
|
|
// return condition when end of string found
|
|
addToken(tokenList, workArea.substring(0, trimStart));
|
|
return -1;
|
|
}
|
|
|
|
/**
|
|
* Unsupported ListIterator operation.
|
|
*
|
|
* @throws UnsupportedOperationException always
|
|
*/
|
|
@Override
|
|
public void remove() {
|
|
throw new UnsupportedOperationException("remove() is unsupported");
|
|
}
|
|
|
|
/**
|
|
* Resets this tokenizer, forgetting all parsing and iteration already completed.
|
|
* <p>
|
|
* This method allows the same tokenizer to be reused for the same String.
|
|
*
|
|
* @return this, to enable chaining
|
|
*/
|
|
public StrTokenizer reset() {
|
|
tokenPos = 0;
|
|
tokens = null;
|
|
return this;
|
|
}
|
|
|
|
/**
|
|
* Reset this tokenizer, giving it a new input string to parse.
|
|
* In this manner you can re-use a tokenizer with the same settings
|
|
* on multiple input lines.
|
|
*
|
|
* @param input the new character array to tokenize, not cloned, null sets no text to parse
|
|
* @return this, to enable chaining
|
|
*/
|
|
public StrTokenizer reset(final char[] input) {
|
|
reset();
|
|
if (input != null) {
|
|
this.chars = input.clone();
|
|
} else {
|
|
this.chars = null;
|
|
}
|
|
return this;
|
|
}
|
|
|
|
/**
|
|
* Reset this tokenizer, giving it a new input string to parse.
|
|
* In this manner you can re-use a tokenizer with the same settings
|
|
* on multiple input lines.
|
|
*
|
|
* @param input the new string to tokenize, null sets no text to parse
|
|
* @return this, to enable chaining
|
|
*/
|
|
public StrTokenizer reset(final String input) {
|
|
reset();
|
|
if (input != null) {
|
|
this.chars = input.toCharArray();
|
|
} else {
|
|
this.chars = null;
|
|
}
|
|
return this;
|
|
}
|
|
|
|
/**
|
|
* Unsupported ListIterator operation.
|
|
* @param obj this parameter ignored.
|
|
* @throws UnsupportedOperationException always
|
|
*/
|
|
@Override
|
|
public void set(final String obj) {
|
|
throw new UnsupportedOperationException("set() is unsupported");
|
|
}
|
|
|
|
/**
|
|
* Sets the field delimiter character.
|
|
*
|
|
* @param delim the delimiter character to use
|
|
* @return this, to enable chaining
|
|
*/
|
|
public StrTokenizer setDelimiterChar(final char delim) {
|
|
return setDelimiterMatcher(StrMatcher.charMatcher(delim));
|
|
}
|
|
|
|
/**
|
|
* Sets the field delimiter matcher.
|
|
* <p>
|
|
* The delimiter is used to separate one token from another.
|
|
* </p>
|
|
*
|
|
* @param delim the delimiter matcher to use
|
|
* @return this, to enable chaining
|
|
*/
|
|
public StrTokenizer setDelimiterMatcher(final StrMatcher delim) {
|
|
if (delim == null) {
|
|
this.delimMatcher = StrMatcher.noneMatcher();
|
|
} else {
|
|
this.delimMatcher = delim;
|
|
}
|
|
return this;
|
|
}
|
|
|
|
/**
|
|
* Sets the field delimiter string.
|
|
*
|
|
* @param delim the delimiter string to use
|
|
* @return this, to enable chaining
|
|
*/
|
|
public StrTokenizer setDelimiterString(final String delim) {
|
|
return setDelimiterMatcher(StrMatcher.stringMatcher(delim));
|
|
}
|
|
|
|
/**
|
|
* Sets whether the tokenizer should return empty tokens as null.
|
|
* The default for this property is false.
|
|
*
|
|
* @param emptyAsNull whether empty tokens are returned as null
|
|
* @return this, to enable chaining
|
|
*/
|
|
public StrTokenizer setEmptyTokenAsNull(final boolean emptyAsNull) {
|
|
this.emptyAsNull = emptyAsNull;
|
|
return this;
|
|
}
|
|
|
|
/**
|
|
* Set the character to ignore.
|
|
* <p>
|
|
* This character is ignored when parsing the String, unless it is
|
|
* within a quoted region.
|
|
* </p>
|
|
*
|
|
* @param ignored the ignored character to use
|
|
* @return this, to enable chaining
|
|
*/
|
|
public StrTokenizer setIgnoredChar(final char ignored) {
|
|
return setIgnoredMatcher(StrMatcher.charMatcher(ignored));
|
|
}
|
|
|
|
/**
|
|
* Set the matcher for characters to ignore.
|
|
* <p>
|
|
* These characters are ignored when parsing the String, unless they are
|
|
* within a quoted region.
|
|
* </p>
|
|
*
|
|
* @param ignored the ignored matcher to use, null ignored
|
|
* @return this, to enable chaining
|
|
*/
|
|
public StrTokenizer setIgnoredMatcher(final StrMatcher ignored) {
|
|
if (ignored != null) {
|
|
this.ignoredMatcher = ignored;
|
|
}
|
|
return this;
|
|
}
|
|
|
|
/**
|
|
* Sets whether the tokenizer should ignore and not return empty tokens.
|
|
* The default for this property is true.
|
|
*
|
|
* @param ignoreEmptyTokens whether empty tokens are not returned
|
|
* @return this, to enable chaining
|
|
*/
|
|
public StrTokenizer setIgnoreEmptyTokens(final boolean ignoreEmptyTokens) {
|
|
this.ignoreEmptyTokens = ignoreEmptyTokens;
|
|
return this;
|
|
}
|
|
|
|
/**
|
|
* Sets the quote character to use.
|
|
* <p>
|
|
* The quote character is used to wrap data between the tokens.
|
|
* This enables delimiters to be entered as data.
|
|
* </p>
|
|
*
|
|
* @param quote the quote character to use
|
|
* @return this, to enable chaining
|
|
*/
|
|
public StrTokenizer setQuoteChar(final char quote) {
|
|
return setQuoteMatcher(StrMatcher.charMatcher(quote));
|
|
}
|
|
|
|
/**
|
|
* Set the quote matcher to use.
|
|
* <p>
|
|
* The quote character is used to wrap data between the tokens.
|
|
* This enables delimiters to be entered as data.
|
|
* </p>
|
|
*
|
|
* @param quote the quote matcher to use, null ignored
|
|
* @return this, to enable chaining
|
|
*/
|
|
public StrTokenizer setQuoteMatcher(final StrMatcher quote) {
|
|
if (quote != null) {
|
|
this.quoteMatcher = quote;
|
|
}
|
|
return this;
|
|
}
|
|
|
|
/**
|
|
* Sets the matcher for characters to trim.
|
|
* <p>
|
|
* These characters are trimmed off on each side of the delimiter
|
|
* until the token or quote is found.
|
|
* </p>
|
|
*
|
|
* @param trimmer the trimmer matcher to use, null ignored
|
|
* @return this, to enable chaining
|
|
*/
|
|
public StrTokenizer setTrimmerMatcher(final StrMatcher trimmer) {
|
|
if (trimmer != null) {
|
|
this.trimmerMatcher = trimmer;
|
|
}
|
|
return this;
|
|
}
|
|
|
|
/**
|
|
* Gets the number of tokens found in the String.
|
|
*
|
|
* @return The number of matched tokens
|
|
*/
|
|
public int size() {
|
|
checkTokenized();
|
|
return tokens.length;
|
|
}
|
|
|
|
/**
|
|
* Internal method to performs the tokenization.
|
|
* <p>
|
|
* Most users of this class do not need to call this method. This method
|
|
* will be called automatically by other (public) methods when required.
|
|
* </p>
|
|
* <p>
|
|
* This method exists to allow subclasses to add code before or after the
|
|
* tokenization. For example, a subclass could alter the character array,
|
|
* offset or count to be parsed, or call the tokenizer multiple times on
|
|
* multiple strings. It is also be possible to filter the results.
|
|
* </p>
|
|
* <p>
|
|
* {@code StrTokenizer} will always pass a zero offset and a count
|
|
* equal to the length of the array to this method, however a subclass
|
|
* may pass other values, or even an entirely different array.
|
|
* </p>
|
|
*
|
|
* @param srcChars the character array being tokenized, may be null
|
|
* @param offset the start position within the character array, must be valid
|
|
* @param count the number of characters to tokenize, must be valid
|
|
* @return The modifiable list of String tokens, unmodifiable if null array or zero count
|
|
*/
|
|
protected List<String> tokenize(final char[] srcChars, final int offset, final int count) {
|
|
if (srcChars == null || count == 0) {
|
|
return Collections.emptyList();
|
|
}
|
|
final StrBuilder buf = new StrBuilder();
|
|
final List<String> tokenList = new ArrayList<>();
|
|
int pos = offset;
|
|
|
|
// loop around the entire buffer
|
|
while (pos >= 0 && pos < count) {
|
|
// find next token
|
|
pos = readNextToken(srcChars, pos, count, buf, tokenList);
|
|
|
|
// handle case where end of string is a delimiter
|
|
if (pos >= count) {
|
|
addToken(tokenList, StringUtils.EMPTY);
|
|
}
|
|
}
|
|
return tokenList;
|
|
}
|
|
|
|
/**
|
|
* Gets the String content that the tokenizer is parsing.
|
|
*
|
|
* @return The string content being parsed
|
|
*/
|
|
@Override
|
|
public String toString() {
|
|
if (tokens == null) {
|
|
return "StrTokenizer[not tokenized yet]";
|
|
}
|
|
return "StrTokenizer" + getTokenList();
|
|
}
|
|
|
|
}
|