1"python3complete.vim - Omni Completion for python
2" Maintainer: Aaron Griffin <aaronmgriffin@gmail.com>
3" Version: 0.9
4" Last Updated: 18 Jun 2009
5"
6" Roland Puntaier: this file contains adaptations for python3 and is parallel to pythoncomplete.vim
7"
8" Changes
9" TODO:
10" 'info' item output can use some formatting work
11" Add an "unsafe eval" mode, to allow for return type evaluation
12" Complete basic syntax along with import statements
13"   i.e. "import url<c-x,c-o>"
14" Continue parsing on invalid line??
15"
16" v 0.9
17"   * Fixed docstring parsing for classes and functions
18"   * Fixed parsing of *args and **kwargs type arguments
19"   * Better function param parsing to handle things like tuples and
20"     lambda defaults args
21"
22" v 0.8
23"   * Fixed an issue where the FIRST assignment was always used instead of
24"   using a subsequent assignment for a variable
25"   * Fixed a scoping issue when working inside a parameterless function
26"
27"
28" v 0.7
29"   * Fixed function list sorting (_ and __ at the bottom)
30"   * Removed newline removal from docs.  It appears vim handles these better in
31"   recent patches
32"
33" v 0.6:
34"   * Fixed argument completion
35"   * Removed the 'kind' completions, as they are better indicated
36"   with real syntax
37"   * Added tuple assignment parsing (whoops, that was forgotten)
38"   * Fixed import handling when flattening scope
39"
40" v 0.5:
41" Yeah, I skipped a version number - 0.4 was never public.
42"  It was a bugfix version on top of 0.3.  This is a complete
43"  rewrite.
44"
45
46if !has('python3')
47    echo "Error: Required vim compiled with +python3"
48    finish
49endif
50
51function! python3complete#Complete(findstart, base)
52    "findstart = 1 when we need to get the text length
53    if a:findstart == 1
54        let line = getline('.')
55        let idx = col('.')
56        while idx > 0
57            let idx -= 1
58            let c = line[idx]
59            if c =~ '\w'
60                continue
61            elseif ! c =~ '\.'
62                let idx = -1
63                break
64            else
65                break
66            endif
67        endwhile
68
69        return idx
70    "findstart = 0 when we need to return the list of completions
71    else
72        "vim no longer moves the cursor upon completion... fix that
73        let line = getline('.')
74        let idx = col('.')
75        let cword = ''
76        while idx > 0
77            let idx -= 1
78            let c = line[idx]
79            if c =~ '\w' || c =~ '\.'
80                let cword = c . cword
81                continue
82            elseif strlen(cword) > 0 || idx == 0
83                break
84            endif
85        endwhile
86        execute "py3 vimpy3complete('" . cword . "', '" . a:base . "')"
87        return g:python3complete_completions
88    endif
89endfunction
90
91function! s:DefPython()
92py3 << PYTHONEOF
93import sys, tokenize, io, types
94from token import NAME, DEDENT, NEWLINE, STRING
95
96debugstmts=[]
97def dbg(s): debugstmts.append(s)
98def showdbg():
99    for d in debugstmts: print("DBG: %s " % d)
100
101def vimpy3complete(context,match):
102    global debugstmts
103    debugstmts = []
104    try:
105        import vim
106        cmpl = Completer()
107        cmpl.evalsource('\n'.join(vim.current.buffer),vim.eval("line('.')"))
108        all = cmpl.get_completions(context,match)
109        all.sort(key=lambda x:x['abbr'].replace('_','z'))
110        dictstr = '['
111        # have to do this for double quoting
112        for cmpl in all:
113            dictstr += '{'
114            for x in cmpl: dictstr += '"%s":"%s",' % (x,cmpl[x])
115            dictstr += '"icase":0},'
116        if dictstr[-1] == ',': dictstr = dictstr[:-1]
117        dictstr += ']'
118        #dbg("dict: %s" % dictstr)
119        vim.command("silent let g:python3complete_completions = %s" % dictstr)
120        #dbg("Completion dict:\n%s" % all)
121    except vim.error:
122        dbg("VIM Error: %s" % vim.error)
123
124class Completer(object):
125    def __init__(self):
126       self.compldict = {}
127       self.parser = PyParser()
128
129    def evalsource(self,text,line=0):
130        sc = self.parser.parse(text,line)
131        src = sc.get_code()
132        dbg("source: %s" % src)
133        try: exec(src,self.compldict)
134        except: dbg("parser: %s, %s" % (sys.exc_info()[0],sys.exc_info()[1]))
135        for l in sc.locals:
136            try: exec(l,self.compldict)
137            except: dbg("locals: %s, %s [%s]" % (sys.exc_info()[0],sys.exc_info()[1],l))
138
139    def _cleanstr(self,doc):
140        return doc.replace('"',' ').replace("'",' ')
141
142    def get_arguments(self,func_obj):
143        def _ctor(class_ob):
144            try: return class_ob.__init__
145            except AttributeError:
146                for base in class_ob.__bases__:
147                    rc = _ctor(base)
148                    if rc is not None: return rc
149            return None
150
151        arg_offset = 1
152        if type(func_obj) == type: func_obj = _ctor(func_obj)
153        elif type(func_obj) == types.MethodType: arg_offset = 1
154        else: arg_offset = 0
155
156        arg_text=''
157        if type(func_obj) in [types.FunctionType, types.LambdaType,types.MethodType]:
158            try:
159                cd = func_obj.__code__
160                real_args = cd.co_varnames[arg_offset:cd.co_argcount]
161                defaults = func_obj.__defaults__ or []
162                defaults = ["=%s" % name for name in defaults]
163                defaults = [""] * (len(real_args)-len(defaults)) + defaults
164                items = [a+d for a,d in zip(real_args,defaults)]
165                if func_obj.__code__.co_flags & 0x4:
166                    items.append("...")
167                if func_obj.__code__.co_flags & 0x8:
168                    items.append("***")
169                arg_text = (','.join(items)) + ')'
170            except:
171                dbg("arg completion: %s: %s" % (sys.exc_info()[0],sys.exc_info()[1]))
172                pass
173        if len(arg_text) == 0:
174            # The doc string sometimes contains the function signature
175            #  this works for alot of C modules that are part of the
176            #  standard library
177            doc = func_obj.__doc__
178            if doc:
179                doc = doc.lstrip()
180                pos = doc.find('\n')
181                if pos > 0:
182                    sigline = doc[:pos]
183                    lidx = sigline.find('(')
184                    ridx = sigline.find(')')
185                    if lidx > 0 and ridx > 0:
186                        arg_text = sigline[lidx+1:ridx] + ')'
187        if len(arg_text) == 0: arg_text = ')'
188        return arg_text
189
190    def get_completions(self,context,match):
191        #dbg("get_completions('%s','%s')" % (context,match))
192        stmt = ''
193        if context: stmt += str(context)
194        if match: stmt += str(match)
195        try:
196            result = None
197            all = {}
198            ridx = stmt.rfind('.')
199            if len(stmt) > 0 and stmt[-1] == '(':
200                result = eval(_sanitize(stmt[:-1]), self.compldict)
201                doc = result.__doc__
202                if doc is None: doc = ''
203                args = self.get_arguments(result)
204                return [{'word':self._cleanstr(args),'info':self._cleanstr(doc)}]
205            elif ridx == -1:
206                match = stmt
207                all = self.compldict
208            else:
209                match = stmt[ridx+1:]
210                stmt = _sanitize(stmt[:ridx])
211                result = eval(stmt, self.compldict)
212                all = dir(result)
213
214            dbg("completing: stmt:%s" % stmt)
215            completions = []
216
217            try: maindoc = result.__doc__
218            except: maindoc = ' '
219            if maindoc is None: maindoc = ' '
220            for m in all:
221                if m == "_PyCmplNoType": continue #this is internal
222                try:
223                    dbg('possible completion: %s' % m)
224                    if m.find(match) == 0:
225                        if result is None: inst = all[m]
226                        else: inst = getattr(result,m)
227                        try: doc = inst.__doc__
228                        except: doc = maindoc
229                        typestr = str(inst)
230                        if doc is None or doc == '': doc = maindoc
231
232                        wrd = m[len(match):]
233                        c = {'word':wrd, 'abbr':m,  'info':self._cleanstr(doc)}
234                        if "function" in typestr:
235                            c['word'] += '('
236                            c['abbr'] += '(' + self._cleanstr(self.get_arguments(inst))
237                        elif "method" in typestr:
238                            c['word'] += '('
239                            c['abbr'] += '(' + self._cleanstr(self.get_arguments(inst))
240                        elif "module" in typestr:
241                            c['word'] += '.'
242                        elif "type" in typestr:
243                            c['word'] += '('
244                            c['abbr'] += '('
245                        completions.append(c)
246                except:
247                    i = sys.exc_info()
248                    dbg("inner completion: %s,%s [stmt='%s']" % (i[0],i[1],stmt))
249            return completions
250        except:
251            i = sys.exc_info()
252            dbg("completion: %s,%s [stmt='%s']" % (i[0],i[1],stmt))
253            return []
254
255class Scope(object):
256    def __init__(self,name,indent,docstr=''):
257        self.subscopes = []
258        self.docstr = docstr
259        self.locals = []
260        self.parent = None
261        self.name = name
262        self.indent = indent
263
264    def add(self,sub):
265        #print('push scope: [%s@%s]' % (sub.name,sub.indent))
266        sub.parent = self
267        self.subscopes.append(sub)
268        return sub
269
270    def doc(self,str):
271        """ Clean up a docstring """
272        d = str.replace('\n',' ')
273        d = d.replace('\t',' ')
274        while d.find('  ') > -1: d = d.replace('  ',' ')
275        while d[0] in '"\'\t ': d = d[1:]
276        while d[-1] in '"\'\t ': d = d[:-1]
277        dbg("Scope(%s)::docstr = %s" % (self,d))
278        self.docstr = d
279
280    def local(self,loc):
281        self._checkexisting(loc)
282        self.locals.append(loc)
283
284    def copy_decl(self,indent=0):
285        """ Copy a scope's declaration only, at the specified indent level - not local variables """
286        return Scope(self.name,indent,self.docstr)
287
288    def _checkexisting(self,test):
289        "Convienance function... keep out duplicates"
290        if test.find('=') > -1:
291            var = test.split('=')[0].strip()
292            for l in self.locals:
293                if l.find('=') > -1 and var == l.split('=')[0].strip():
294                    self.locals.remove(l)
295
296    def get_code(self):
297        str = ""
298        if len(self.docstr) > 0: str += '"""'+self.docstr+'"""\n'
299        for l in self.locals:
300            if l.startswith('import'): str += l+'\n'
301        str += 'class _PyCmplNoType:\n    def __getattr__(self,name):\n        return None\n'
302        for sub in self.subscopes:
303            str += sub.get_code()
304        for l in self.locals:
305            if not l.startswith('import'): str += l+'\n'
306
307        return str
308
309    def pop(self,indent):
310        #print('pop scope: [%s] to [%s]' % (self.indent,indent))
311        outer = self
312        while outer.parent != None and outer.indent >= indent:
313            outer = outer.parent
314        return outer
315
316    def currentindent(self):
317        #print('parse current indent: %s' % self.indent)
318        return '    '*self.indent
319
320    def childindent(self):
321        #print('parse child indent: [%s]' % (self.indent+1))
322        return '    '*(self.indent+1)
323
324class Class(Scope):
325    def __init__(self, name, supers, indent, docstr=''):
326        Scope.__init__(self,name,indent, docstr)
327        self.supers = supers
328    def copy_decl(self,indent=0):
329        c = Class(self.name,self.supers,indent, self.docstr)
330        for s in self.subscopes:
331            c.add(s.copy_decl(indent+1))
332        return c
333    def get_code(self):
334        str = '%sclass %s' % (self.currentindent(),self.name)
335        if len(self.supers) > 0: str += '(%s)' % ','.join(self.supers)
336        str += ':\n'
337        if len(self.docstr) > 0: str += self.childindent()+'"""'+self.docstr+'"""\n'
338        if len(self.subscopes) > 0:
339            for s in self.subscopes: str += s.get_code()
340        else:
341            str += '%spass\n' % self.childindent()
342        return str
343
344
345class Function(Scope):
346    def __init__(self, name, params, indent, docstr=''):
347        Scope.__init__(self,name,indent, docstr)
348        self.params = params
349    def copy_decl(self,indent=0):
350        return Function(self.name,self.params,indent, self.docstr)
351    def get_code(self):
352        str = "%sdef %s(%s):\n" % \
353            (self.currentindent(),self.name,','.join(self.params))
354        if len(self.docstr) > 0: str += self.childindent()+'"""'+self.docstr+'"""\n'
355        str += "%spass\n" % self.childindent()
356        return str
357
358class PyParser:
359    def __init__(self):
360        self.top = Scope('global',0)
361        self.scope = self.top
362
363    def _parsedotname(self,pre=None):
364        #returns (dottedname, nexttoken)
365        name = []
366        if pre is None:
367            tokentype, token, indent = self.donext()
368            if tokentype != NAME and token != '*':
369                return ('', token)
370        else: token = pre
371        name.append(token)
372        while True:
373            tokentype, token, indent = self.donext()
374            if token != '.': break
375            tokentype, token, indent = self.donext()
376            if tokentype != NAME: break
377            name.append(token)
378        return (".".join(name), token)
379
380    def _parseimportlist(self):
381        imports = []
382        while True:
383            name, token = self._parsedotname()
384            if not name: break
385            name2 = ''
386            if token == 'as': name2, token = self._parsedotname()
387            imports.append((name, name2))
388            while token != "," and "\n" not in token:
389                tokentype, token, indent = self.donext()
390            if token != ",": break
391        return imports
392
393    def _parenparse(self):
394        name = ''
395        names = []
396        level = 1
397        while True:
398            tokentype, token, indent = self.donext()
399            if token in (')', ',') and level == 1:
400                if '=' not in name: name = name.replace(' ', '')
401                names.append(name.strip())
402                name = ''
403            if token == '(':
404                level += 1
405                name += "("
406            elif token == ')':
407                level -= 1
408                if level == 0: break
409                else: name += ")"
410            elif token == ',' and level == 1:
411                pass
412            else:
413                name += "%s " % str(token)
414        return names
415
416    def _parsefunction(self,indent):
417        self.scope=self.scope.pop(indent)
418        tokentype, fname, ind = self.donext()
419        if tokentype != NAME: return None
420
421        tokentype, open, ind = self.donext()
422        if open != '(': return None
423        params=self._parenparse()
424
425        tokentype, colon, ind = self.donext()
426        if colon != ':': return None
427
428        return Function(fname,params,indent)
429
430    def _parseclass(self,indent):
431        self.scope=self.scope.pop(indent)
432        tokentype, cname, ind = self.donext()
433        if tokentype != NAME: return None
434
435        super = []
436        tokentype, thenext, ind = self.donext()
437        if thenext == '(':
438            super=self._parenparse()
439        elif thenext != ':': return None
440
441        return Class(cname,super,indent)
442
443    def _parseassignment(self):
444        assign=''
445        tokentype, token, indent = self.donext()
446        if tokentype == tokenize.STRING or token == 'str':  
447            return '""'
448        elif token == '(' or token == 'tuple':
449            return '()'
450        elif token == '[' or token == 'list':
451            return '[]'
452        elif token == '{' or token == 'dict':
453            return '{}'
454        elif tokentype == tokenize.NUMBER:
455            return '0'
456        elif token == 'open' or token == 'file':
457            return 'file'
458        elif token == 'None':
459            return '_PyCmplNoType()'
460        elif token == 'type':
461            return 'type(_PyCmplNoType)' #only for method resolution
462        else:
463            assign += token
464            level = 0
465            while True:
466                tokentype, token, indent = self.donext()
467                if token in ('(','{','['):
468                    level += 1
469                elif token in (']','}',')'):
470                    level -= 1
471                    if level == 0: break
472                elif level == 0:
473                    if token in (';','\n'): break
474                    assign += token
475        return "%s" % assign
476
477    def donext(self):
478        type, token, (lineno, indent), end, self.parserline = next(self.gen)
479        if lineno == self.curline:
480            #print('line found [%s] scope=%s' % (line.replace('\n',''),self.scope.name))
481            self.currentscope = self.scope
482        return (type, token, indent)
483
484    def _adjustvisibility(self):
485        newscope = Scope('result',0)
486        scp = self.currentscope
487        while scp != None:
488            if type(scp) == Function:
489                slice = 0
490                #Handle 'self' params
491                if scp.parent != None and type(scp.parent) == Class:
492                    slice = 1
493                    newscope.local('%s = %s' % (scp.params[0],scp.parent.name))
494                for p in scp.params[slice:]:
495                    i = p.find('=')
496                    if len(p) == 0: continue
497                    pvar = ''
498                    ptype = ''
499                    if i == -1:
500                        pvar = p
501                        ptype = '_PyCmplNoType()'
502                    else:
503                        pvar = p[:i]
504                        ptype = _sanitize(p[i+1:])
505                    if pvar.startswith('**'):
506                        pvar = pvar[2:]
507                        ptype = '{}'
508                    elif pvar.startswith('*'):
509                        pvar = pvar[1:]
510                        ptype = '[]'
511
512                    newscope.local('%s = %s' % (pvar,ptype))
513
514            for s in scp.subscopes:
515                ns = s.copy_decl(0)
516                newscope.add(ns)
517            for l in scp.locals: newscope.local(l)
518            scp = scp.parent
519
520        self.currentscope = newscope
521        return self.currentscope
522
523    #p.parse(vim.current.buffer[:],vim.eval("line('.')"))
524    def parse(self,text,curline=0):
525        self.curline = int(curline)
526        buf = io.StringIO(''.join(text) + '\n')
527        self.gen = tokenize.generate_tokens(buf.readline)
528        self.currentscope = self.scope
529
530        try:
531            freshscope=True
532            while True:
533                tokentype, token, indent = self.donext()
534                #dbg( 'main: token=[%s] indent=[%s]' % (token,indent))
535
536                if tokentype == DEDENT or token == "pass":
537                    self.scope = self.scope.pop(indent)
538                elif token == 'def':
539                    func = self._parsefunction(indent)
540                    if func is None:
541                        print("function: syntax error...")
542                        continue
543                    dbg("new scope: function")
544                    freshscope = True
545                    self.scope = self.scope.add(func)
546                elif token == 'class':
547                    cls = self._parseclass(indent)
548                    if cls is None:
549                        print("class: syntax error...")
550                        continue
551                    freshscope = True
552                    dbg("new scope: class")
553                    self.scope = self.scope.add(cls)
554                    
555                elif token == 'import':
556                    imports = self._parseimportlist()
557                    for mod, alias in imports:
558                        loc = "import %s" % mod
559                        if len(alias) > 0: loc += " as %s" % alias
560                        self.scope.local(loc)
561                    freshscope = False
562                elif token == 'from':
563                    mod, token = self._parsedotname()
564                    if not mod or token != "import":
565                        print("from: syntax error...")
566                        continue
567                    names = self._parseimportlist()
568                    for name, alias in names:
569                        loc = "from %s import %s" % (mod,name)
570                        if len(alias) > 0: loc += " as %s" % alias
571                        self.scope.local(loc)
572                    freshscope = False
573                elif tokentype == STRING:
574                    if freshscope: self.scope.doc(token)
575                elif tokentype == NAME:
576                    name,token = self._parsedotname(token) 
577                    if token == '=':
578                        stmt = self._parseassignment()
579                        dbg("parseassignment: %s = %s" % (name, stmt))
580                        if stmt != None:
581                            self.scope.local("%s = %s" % (name,stmt))
582                    freshscope = False
583        except StopIteration: #thrown on EOF
584            pass
585        except:
586            dbg("parse error: %s, %s @ %s" %
587                (sys.exc_info()[0], sys.exc_info()[1], self.parserline))
588        return self._adjustvisibility()
589
590def _sanitize(str):
591    val = ''
592    level = 0
593    for c in str:
594        if c in ('(','{','['):
595            level += 1
596        elif c in (']','}',')'):
597            level -= 1
598        elif level == 0:
599            val += c
600    return val
601
602sys.path.extend(['.','..'])
603PYTHONEOF
604endfunction
605
606call s:DefPython()
607