shlex.py (9727B)
1 #Embedded file name: C:\BuildAgent\work\20d73f1b24eed06e\eve\release\V24.01\eve\common\lib\shlex.py 2 import os.path 3 import sys 4 from collections import deque 5 try: 6 from cStringIO import StringIO 7 except ImportError: 8 from StringIO import StringIO 9 10 __all__ = ['shlex', 'split'] 11 12 class shlex: 13 14 def __init__(self, instream = None, infile = None, posix = False): 15 if isinstance(instream, basestring): 16 instream = StringIO(instream) 17 if instream is not None: 18 self.instream = instream 19 self.infile = infile 20 else: 21 self.instream = sys.stdin 22 self.infile = None 23 self.posix = posix 24 if posix: 25 self.eof = None 26 else: 27 self.eof = '' 28 self.commenters = '#' 29 self.wordchars = 'abcdfeghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_' 30 if self.posix: 31 self.wordchars += '\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd8\xd9\xda\xdb\xdc\xdd\xde' 32 self.whitespace = ' \t\r\n' 33 self.whitespace_split = False 34 self.quotes = '\'"' 35 self.escape = '\\' 36 self.escapedquotes = '"' 37 self.state = ' ' 38 self.pushback = deque() 39 self.lineno = 1 40 self.debug = 0 41 self.token = '' 42 self.filestack = deque() 43 self.source = None 44 if self.debug: 45 print 'shlex: reading from %s, line %d' % (self.instream, self.lineno) 46 47 def push_token(self, tok): 48 if self.debug >= 1: 49 print 'shlex: pushing token ' + repr(tok) 50 self.pushback.appendleft(tok) 51 52 def push_source(self, newstream, newfile = None): 53 if isinstance(newstream, basestring): 54 newstream = StringIO(newstream) 55 self.filestack.appendleft((self.infile, self.instream, self.lineno)) 56 self.infile = newfile 57 self.instream = newstream 58 self.lineno = 1 59 if self.debug: 60 if newfile is not None: 61 print 'shlex: pushing to file %s' % (self.infile,) 62 else: 63 print 'shlex: pushing to stream %s' % (self.instream,) 64 65 def pop_source(self): 66 self.instream.close() 67 self.infile, self.instream, self.lineno = self.filestack.popleft() 68 if self.debug: 69 print 'shlex: popping to %s, line %d' % (self.instream, self.lineno) 70 self.state = ' ' 71 72 def get_token(self): 73 if self.pushback: 74 tok = self.pushback.popleft() 75 if self.debug >= 1: 76 print 'shlex: popping token ' + repr(tok) 77 return tok 78 raw = self.read_token() 79 if self.source is not None: 80 while raw == self.source: 81 spec = self.sourcehook(self.read_token()) 82 if spec: 83 newfile, newstream = spec 84 self.push_source(newstream, newfile) 85 raw = self.get_token() 86 87 while raw == self.eof: 88 if not self.filestack: 89 return self.eof 90 self.pop_source() 91 raw = self.get_token() 92 93 if self.debug >= 1: 94 if raw != self.eof: 95 print 'shlex: token=' + repr(raw) 96 else: 97 print 'shlex: token=EOF' 98 return raw 99 100 def read_token(self): 101 quoted = False 102 escapedstate = ' ' 103 while True: 104 nextchar = self.instream.read(1) 105 if nextchar == '\n': 106 self.lineno = self.lineno + 1 107 if self.debug >= 3: 108 print 'shlex: in state', repr(self.state), 109 print 'I see character:', repr(nextchar) 110 if self.state is None: 111 self.token = '' 112 break 113 elif self.state == ' ': 114 if not nextchar: 115 self.state = None 116 break 117 elif nextchar in self.whitespace: 118 if self.debug >= 2: 119 print 'shlex: I see whitespace in whitespace state' 120 if self.token or self.posix and quoted: 121 break 122 else: 123 continue 124 elif nextchar in self.commenters: 125 self.instream.readline() 126 self.lineno = self.lineno + 1 127 elif self.posix and nextchar in self.escape: 128 escapedstate = 'a' 129 self.state = nextchar 130 elif nextchar in self.wordchars: 131 self.token = nextchar 132 self.state = 'a' 133 elif nextchar in self.quotes: 134 if not self.posix: 135 self.token = nextchar 136 self.state = nextchar 137 elif self.whitespace_split: 138 self.token = nextchar 139 self.state = 'a' 140 else: 141 self.token = nextchar 142 if self.token or self.posix and quoted: 143 break 144 else: 145 continue 146 elif self.state in self.quotes: 147 quoted = True 148 if not nextchar: 149 if self.debug >= 2: 150 print 'shlex: I see EOF in quotes state' 151 raise ValueError, 'No closing quotation' 152 if nextchar == self.state: 153 if not self.posix: 154 self.token = self.token + nextchar 155 self.state = ' ' 156 break 157 else: 158 self.state = 'a' 159 elif self.posix and nextchar in self.escape and self.state in self.escapedquotes: 160 escapedstate = self.state 161 self.state = nextchar 162 else: 163 self.token = self.token + nextchar 164 elif self.state in self.escape: 165 if not nextchar: 166 if self.debug >= 2: 167 print 'shlex: I see EOF in escape state' 168 raise ValueError, 'No escaped character' 169 if escapedstate in self.quotes and nextchar != self.state and nextchar != escapedstate: 170 self.token = self.token + self.state 171 self.token = self.token + nextchar 172 self.state = escapedstate 173 elif self.state == 'a': 174 if not nextchar: 175 self.state = None 176 break 177 elif nextchar in self.whitespace: 178 if self.debug >= 2: 179 print 'shlex: I see whitespace in word state' 180 self.state = ' ' 181 if self.token or self.posix and quoted: 182 break 183 else: 184 continue 185 elif nextchar in self.commenters: 186 self.instream.readline() 187 self.lineno = self.lineno + 1 188 if self.posix: 189 self.state = ' ' 190 if self.token or self.posix and quoted: 191 break 192 else: 193 continue 194 elif self.posix and nextchar in self.quotes: 195 self.state = nextchar 196 elif self.posix and nextchar in self.escape: 197 escapedstate = 'a' 198 self.state = nextchar 199 elif nextchar in self.wordchars or nextchar in self.quotes or self.whitespace_split: 200 self.token = self.token + nextchar 201 else: 202 self.pushback.appendleft(nextchar) 203 if self.debug >= 2: 204 print 'shlex: I see punctuation in word state' 205 self.state = ' ' 206 if self.token: 207 break 208 else: 209 continue 210 211 result = self.token 212 self.token = '' 213 if self.posix and not quoted and result == '': 214 result = None 215 if self.debug > 1: 216 if result: 217 print 'shlex: raw token=' + repr(result) 218 else: 219 print 'shlex: raw token=EOF' 220 return result 221 222 def sourcehook(self, newfile): 223 if newfile[0] == '"': 224 newfile = newfile[1:-1] 225 if isinstance(self.infile, basestring) and not os.path.isabs(newfile): 226 newfile = os.path.join(os.path.dirname(self.infile), newfile) 227 return (newfile, open(newfile, 'r')) 228 229 def error_leader(self, infile = None, lineno = None): 230 if infile is None: 231 infile = self.infile 232 if lineno is None: 233 lineno = self.lineno 234 return '"%s", line %d: ' % (infile, lineno) 235 236 def __iter__(self): 237 return self 238 239 def next(self): 240 token = self.get_token() 241 if token == self.eof: 242 raise StopIteration 243 return token 244 245 246 def split(s, comments = False, posix = True): 247 lex = shlex(s, posix=posix) 248 lex.whitespace_split = True 249 if not comments: 250 lex.commenters = '' 251 return list(lex) 252 253 254 if __name__ == '__main__': 255 if len(sys.argv) == 1: 256 lexer = shlex() 257 else: 258 file = sys.argv[1] 259 lexer = shlex(open(file), file) 260 while 1: 261 tt = lexer.get_token() 262 if tt: 263 print 'Token: ' + repr(tt) 264 else: 265 break