Millet Porridge

English version of https://corvo.myseu.cn

0%

Some Debugging of YouCompleteMe (Python)

Posted on Edited on In Disqus:
YouCompleteMe is a Vim completion plugin that also uses a client-server mechanism. I recently spent time debugging and examining its code; I'll share some of the patching process as a small note, to be improved if needed.

As for the occasion, I think it was seeing Microsoft recently open-source a new static analysis tool pyright — a static analysis tool for Python; such tools are mainly used for syntax checking and completion. I have been using YouCompleteMe all along, so spending some time to see how it works is worthwhile. This post will be updated often; I’ll share some thoughts from my debugging process.

Basic Principle

When using YouCompleteMe, a ycmd process auto-starts in the background — this is the server side. When completing, the relevant code snippet is sent to the server, which returns possible completions.

Under the plugin directory (~/.vim/plugged/YouCompleteMe):

  1. the server side is in third_party/ycmd
  2. the client side is in python/ycm

The two communicate via the HTTP protocol.

A Few Patches

Since YouCompleteMe starts ycmd, I’ll just modify that part of the code; we manually start ycmd to debug the server side.

Client-side patch

This patch mainly does 4 things:

  1. Fixes the hmac_secret so it doesn’t need regenerating each time when debugging;
  2. Fixes the local HTTP port (30827 in this post);
  3. Comments out utils.SafePopen below so ycmd isn’t started directly when vim starts;
  4. Doesn’t generate a temporary config file
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
diff --git a/python/ycm/youcompleteme.py b/python/ycm/youcompleteme.py
index 530d7d13..ee8cf77c 100644
--- a/python/ycm/youcompleteme.py
+++ b/python/ycm/youcompleteme.py
@@ -139,7 +139,8 @@ class YouCompleteMe( object ):
 
     self._SetLogLevel()
 
-    hmac_secret = os.urandom( HMAC_SECRET_LENGTH )
+    # hmac_secret = os.urandom( HMAC_SECRET_LENGTH )
+    hmac_secret = b"hello world"
     options_dict = dict( self._user_options )
     options_dict[ 'hmac_secret' ] = utils.ToUnicode(
       base64.b64encode( hmac_secret ) )
@@ -147,10 +148,13 @@ class YouCompleteMe( object ):
       'keep_logfiles' ]
 
     # The temp options file is deleted by ycmd during startup.
-    with NamedTemporaryFile( delete = False, mode = 'w+' ) as options_file:
-      json.dump( options_dict, options_file )
+    # with NamedTemporaryFile( delete = False, mode = 'w+' ) as options_file:
+    #  json.dump( options_dict, options_file )
 
-    server_port = utils.GetUnusedLocalhostPort()
+    # print(options_dict)
+    # server_port = utils.GetUnusedLocalhostPort()
+    server_port = 30827
+    # print(options_file.name)
 
     BaseRequest.server_location = 'http://127.0.0.1:' + str( server_port )
     BaseRequest.hmac_secret = hmac_secret
@@ -169,7 +173,7 @@ class YouCompleteMe( object ):
     args = [ python_interpreter,
              paths.PathToServerScript(),
              '--port={0}'.format( server_port ),
-             '--options_file={0}'.format( options_file.name ),
+    #          '--options_file={0}'.format( options_file.name ),
              '--log={0}'.format( self._user_options[ 'log_level' ] ),
              '--idle_suicide_seconds={0}'.format(
                 SERVER_IDLE_SUICIDE_SECONDS ) ]
@@ -183,9 +187,9 @@ class YouCompleteMe( object ):
 
     if self._user_options[ 'keep_logfiles' ]:
       args.append( '--keep_logfiles' )
-
-    self._server_popen = utils.SafePopen( args, stdin_windows = PIPE,
-                                          stdout = PIPE, stderr = PIPE )
+    print(args)
+    # self._server_popen = utils.SafePopen( args, stdin_windows = PIPE,
+    #                                      stdout = PIPE, stderr = PIPE )
 
 
   def _SetUpLogging( self ):

Server-side patch

This patch does 2 things:

  1. Prevents ycmd from deleting the options_file on exit — using our own file, it must not be deleted.
  2. On receiving the shutdown signal, don’t shut down, to avoid constantly restarting the server side.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
diff --git a/ycmd/__main__.py b/ycmd/__main__.py
index 39e14f72..33ce635c 100644
--- a/ycmd/__main__.py
+++ b/ycmd/__main__.py
@@ -135,7 +135,7 @@ def SetupOptions( options_file ):
   options = user_options_store.DefaultOptions()
   user_options = json.loads( ReadFile( options_file ) )
   options.update( user_options )
-  utils.RemoveIfExists( options_file )
+  # utils.RemoveIfExists( options_file )
   hmac_secret = ToBytes( base64.b64decode( options[ 'hmac_secret' ] ) )
   del options[ 'hmac_secret' ]
   user_options_store.SetAll( options )
diff --git a/ycmd/handlers.py b/ycmd/handlers.py
index 38ee87e2..18fcdcb3 100644
--- a/ycmd/handlers.py
+++ b/ycmd/handlers.py
@@ -241,7 +241,7 @@ def DebugInfo():
 @app.post( '/shutdown' )
 def Shutdown():
   LOGGER.info( 'Received shutdown request' )
-  ServerShutdown()
+  # ServerShutdown()
 
   return _JsonResponse( True )

Manually starting ycmd

When starting, specify the options_file and our pre-assigned port:

1
2
3
4
> /usr/bin/python  \
    ~/.vim/plugged/YouCompleteMe/python/ycm/../../third_party/ycmd/ycmd \
    --port=30827 \
    --options_file=ycmd.json

ycmd.json — this file is generated by ycm; I saved a separate copy here that can be used directly

TL; DR click to expand 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
{
  "filepath_blacklist": {
    "jsx": "1",
    "xml": "1",
    "html": "1"
  },
  "clangd_uses_ycmd_caching": 1,
  "autoclose_preview_window_after_insertion": 0,
  "collect_identifiers_from_tags_files": 0,
  "autoclose_preview_window_after_completion": 1,
  "filetype_blacklist": {
    "unite": "1"
  },
  "key_list_previous_completion": [
    "<S-TAB>",
    "<Up>"
  ],
  "seed_identifiers_with_syntax": 0,
  "auto_stop_csharp_server": 1,
  "show_diagnostics_ui": 1,
  "disable_for_files_larger_than_kb": 1000,
  "clangd_args": [],
  "collect_identifiers_from_comments_and_strings": 0,
  "max_diagnostics_to_display": 30,
  "complete_in_strings": 1,
  "filetype_specific_completion_to_disable": {
    "gitcommit": "1"
  },
  "max_num_identifier_candidates": 10,
  "open_loclist_on_ycm_diags": 1,
  "extra_conf_vim_data": [],
  "keep_logfiles": 0,
  "use_ultisnips_completer": 1,
  "min_num_of_chars_for_completion": 1,
  "enable_diagnostic_highlighting": 1,
  "key_detailed_diagnostics": "<leader>d",
  "warning_symbol": ">>",
  "auto_trigger": 1,
  "goto_buffer_command": "same-buffer",
  "echo_current_diagnostic": 1,
  "confirm_extra_conf": 1,
  "gocode_binary_path": "",
  "racerd_binary_path": "",
  "filter_diagnostics": {},
  "rust_src_path": "",
  "python_binary_path": "/usr/bin/python3",
  "java_jdtls_use_clean_workspace": 1,
  "clangd_binary_path": "",
  "server_python_interpreter": "",
  "max_num_candidates": 50,
  "min_identifier_candidate_chars": 0,
  "always_populate_location_list": 0,
  "add_preview_to_completeopt": 0,
  "global_ycm_extra_conf": "~/.vim/ycm_extra_conf.py",
  "enable_diagnostic_signs": 1,
  "use_clangd": 1,
  "key_list_select_completion": [
    "<TAB>",
    "<Down>"
  ],
  "extra_conf_globlist": [],
  "log_level": "info",
  "complete_in_comments": 0,
  "key_invoke_completion": "<C-Space>",
  "filepath_completion_use_working_dir": 0,
  "auto_start_csharp_server": 1,
  "cache_omnifunc": 1,
  "filetype_whitelist": {
    "*": "1"
  },
  "csharp_server_port": 0,
  "key_list_stop_completion": [
    "<C-y>"
  ],
  "error_symbol": ">>",
  "godef_binary_path": "",
  "semantic_triggers": {
    "c": [
      "->",
      "."
    ],
    "perl": [
      "->"
    ],
    "cpp,objcpp": [
      "->",
      ".",
      "::"
    ],
    "lua": [
      ".",
      ":"
    ],
    "php": [
      "->",
      "::"
    ],
    "objc": [
      "->",
      "."
    ],
    "cs,java,javascript,d,vim,ruby,python,perl6,scala,vb,elixir,go": [
      "."
    ],
    "erlang": [
      ":"
    ]
  },
  "hmac_secret": "aGVsbG8gd29ybGQ=",
  "server_keep_logfiles": 0
}

Debugging with Tornado

ycmd uses the bottle framework; it can also be debugged with tornado. Everyone is welcome to try it yourselves.

Related Posts