To: vim_dev@googlegroups.com Subject: Patch 8.2.0677 Fcc: outbox From: Bram Moolenaar Mime-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ------------ Patch 8.2.0677 Problem: Vim9: no support for closures. Solution: Find variables in the outer function scope, so long as the scope exists. Files: src/vim9compile.c, src/proto/vim9compile.pro, src/userfunc.c, src/vim9execute.c, src/structs.h, src/vim9.h, src/testdir/test_vim9_func.vim *** ../vim-8.2.0676/src/vim9compile.c 2020-05-01 15:44:24.535895262 +0200 --- src/vim9compile.c 2020-05-01 19:15:16.865897958 +0200 *************** *** 97,105 **** typedef struct { char_u *lv_name; type_T *lv_type; ! int lv_idx; // index of the variable on the stack ! int lv_const; // when TRUE cannot be assigned to ! int lv_arg; // when TRUE this is an argument } lvar_T; /* --- 97,106 ---- typedef struct { char_u *lv_name; type_T *lv_type; ! int lv_idx; // index of the variable on the stack ! int lv_from_outer; // when TRUE using ctx_outer scope ! int lv_const; // when TRUE cannot be assigned to ! int lv_arg; // when TRUE this is an argument } lvar_T; /* *************** *** 123,128 **** --- 124,130 ---- cctx_T *ctx_outer; // outer scope for lambda or nested // function + int ctx_outer_used; // var in ctx_outer was used garray_T ctx_type_stack; // type of each item on the stack garray_T *ctx_type_list; // list of pointers to allocated types *************** *** 146,162 **** lookup_local(char_u *name, size_t len, cctx_T *cctx) { int idx; if (len == 0) return NULL; for (idx = 0; idx < cctx->ctx_locals.ga_len; ++idx) { ! lvar_T *lvar = ((lvar_T *)cctx->ctx_locals.ga_data) + idx; ! if (STRNCMP(name, lvar->lv_name, len) == 0 && STRLEN(lvar->lv_name) == len) return lvar; } return NULL; } --- 148,184 ---- lookup_local(char_u *name, size_t len, cctx_T *cctx) { int idx; + lvar_T *lvar; if (len == 0) return NULL; + + // Find local in current function scope. for (idx = 0; idx < cctx->ctx_locals.ga_len; ++idx) { ! lvar = ((lvar_T *)cctx->ctx_locals.ga_data) + idx; if (STRNCMP(name, lvar->lv_name, len) == 0 && STRLEN(lvar->lv_name) == len) + { + lvar->lv_from_outer = FALSE; return lvar; + } } + + // Find local in outer function scope. + if (cctx->ctx_outer != NULL) + { + lvar = lookup_local(name, len, cctx->ctx_outer); + if (lvar != NULL) + { + // TODO: are there situations we should not mark the outer scope as + // used? + cctx->ctx_outer_used = TRUE; + lvar->lv_from_outer = TRUE; + return lvar; + } + } + return NULL; } *************** *** 417,422 **** --- 439,509 ---- return &t_any; // not used } + static void + type_mismatch(type_T *expected, type_T *actual) + { + char *tofree1, *tofree2; + + semsg(_("E1013: type mismatch, expected %s but got %s"), + type_name(expected, &tofree1), type_name(actual, &tofree2)); + vim_free(tofree1); + vim_free(tofree2); + } + + static void + arg_type_mismatch(type_T *expected, type_T *actual, int argidx) + { + char *tofree1, *tofree2; + + semsg(_("E1013: argument %d: type mismatch, expected %s but got %s"), + argidx, + type_name(expected, &tofree1), type_name(actual, &tofree2)); + vim_free(tofree1); + vim_free(tofree2); + } + + /* + * Check if the expected and actual types match. + * Does not allow for assigning "any" to a specific type. + */ + static int + check_type(type_T *expected, type_T *actual, int give_msg) + { + int ret = OK; + + // When expected is "unknown" we accept any actual type. + // When expected is "any" we accept any actual type except "void". + if (expected->tt_type != VAR_UNKNOWN + && !(expected->tt_type == VAR_ANY && actual->tt_type != VAR_VOID)) + + { + if (expected->tt_type != actual->tt_type) + { + if (give_msg) + type_mismatch(expected, actual); + return FAIL; + } + if (expected->tt_type == VAR_DICT || expected->tt_type == VAR_LIST) + { + // "unknown" is used for an empty list or dict + if (actual->tt_member != &t_unknown) + ret = check_type(expected->tt_member, actual->tt_member, FALSE); + } + else if (expected->tt_type == VAR_FUNC) + { + if (expected->tt_member != &t_unknown) + ret = check_type(expected->tt_member, actual->tt_member, FALSE); + if (ret == OK && expected->tt_argcount != -1 + && (actual->tt_argcount < expected->tt_min_argcount + || actual->tt_argcount > expected->tt_argcount)) + ret = FAIL; + } + if (ret == FAIL && give_msg) + type_mismatch(expected, actual); + } + return ret; + } + ///////////////////////////////////////////////////////////////////// // Following generate_ functions expect the caller to call ga_grow(). *************** *** 740,745 **** --- 827,855 ---- } /* + * Check that + * - "actual" is "expected" type or + * - "actual" is a type that can be "expected" type: add a runtime check; or + * - return FAIL. + */ + static int + need_type(type_T *actual, type_T *expected, int offset, cctx_T *cctx) + { + if (check_type(expected, actual, FALSE) == OK) + return OK; + if (actual->tt_type != VAR_ANY + && actual->tt_type != VAR_UNKNOWN + && !(actual->tt_type == VAR_FUNC + && (actual->tt_member == &t_any || actual->tt_argcount < 0))) + { + type_mismatch(expected, actual); + return FAIL; + } + generate_TYPECHECK(cctx, expected, offset); + return OK; + } + + /* * Generate an ISN_PUSHNR instruction. */ static int *************** *** 1272,1278 **** else expected = ufunc->uf_va_type->tt_member; actual = ((type_T **)stack->ga_data)[stack->ga_len - argcount + i]; ! if (check_type(expected, actual, FALSE) == FAIL) { arg_type_mismatch(expected, actual, i + 1); return FAIL; --- 1382,1388 ---- else expected = ufunc->uf_va_type->tt_member; actual = ((type_T **)stack->ga_data)[stack->ga_len - argcount + i]; ! if (need_type(actual, expected, -argcount + i, cctx) == FAIL) { arg_type_mismatch(expected, actual, i + 1); return FAIL; *************** *** 1543,1548 **** --- 1653,1672 ---- if (*p == '>') ++p; } + else if (*p == '(' && STRNCMP("func", start, 4) == 0) + { + // handle func(args): type + ++p; + while (*p != ')' && *p != NUL) + { + p = skip_type(p); + if (*p == ',') + p = skipwhite(p + 1); + } + if (*p == ')' && p[1] == ':') + p = skip_type(skipwhite(p + 2)); + } + return p; } *************** *** 2309,2314 **** --- 2433,2439 ---- size_t len = end - *arg; int idx; int gen_load = FALSE; + int gen_load_outer = FALSE; name = vim_strnsave(*arg, end - *arg); if (name == NULL) *************** *** 2343,2349 **** { type = lvar->lv_type; idx = lvar->lv_idx; ! gen_load = TRUE; } else { --- 2468,2477 ---- { type = lvar->lv_type; idx = lvar->lv_idx; ! if (lvar->lv_from_outer) ! gen_load_outer = TRUE; ! else ! gen_load = TRUE; } else { *************** *** 2370,2375 **** --- 2498,2505 ---- } if (gen_load) res = generate_LOAD(cctx, ISN_LOAD, idx, NULL, type); + if (gen_load_outer) + res = generate_LOAD(cctx, ISN_LOADOUTER, idx, NULL, type); } *arg = end; *************** *** 2578,2671 **** return p; } - static void - type_mismatch(type_T *expected, type_T *actual) - { - char *tofree1, *tofree2; - - semsg(_("E1013: type mismatch, expected %s but got %s"), - type_name(expected, &tofree1), type_name(actual, &tofree2)); - vim_free(tofree1); - vim_free(tofree2); - } - - static void - arg_type_mismatch(type_T *expected, type_T *actual, int argidx) - { - char *tofree1, *tofree2; - - semsg(_("E1013: argument %d: type mismatch, expected %s but got %s"), - argidx, - type_name(expected, &tofree1), type_name(actual, &tofree2)); - vim_free(tofree1); - vim_free(tofree2); - } - - /* - * Check if the expected and actual types match. - * Does not allow for assigning "any" to a specific type. - */ - static int - check_type(type_T *expected, type_T *actual, int give_msg) - { - int ret = OK; - - // When expected is "unknown" we accept any actual type. - // When expected is "any" we accept any actual type except "void". - if (expected->tt_type != VAR_UNKNOWN - && !(expected->tt_type == VAR_ANY && actual->tt_type != VAR_VOID)) - - { - if (expected->tt_type != actual->tt_type) - { - if (give_msg) - type_mismatch(expected, actual); - return FAIL; - } - if (expected->tt_type == VAR_DICT || expected->tt_type == VAR_LIST) - { - // "unknown" is used for an empty list or dict - if (actual->tt_member != &t_unknown) - ret = check_type(expected->tt_member, actual->tt_member, FALSE); - } - else if (expected->tt_type == VAR_FUNC) - { - if (expected->tt_member != &t_unknown) - ret = check_type(expected->tt_member, actual->tt_member, FALSE); - if (ret == OK && expected->tt_argcount != -1 - && (actual->tt_argcount < expected->tt_min_argcount - || actual->tt_argcount > expected->tt_argcount)) - ret = FAIL; - } - if (ret == FAIL && give_msg) - type_mismatch(expected, actual); - } - return ret; - } - - /* - * Check that - * - "actual" is "expected" type or - * - "actual" is a type that can be "expected" type: add a runtime check; or - * - return FAIL. - */ - static int - need_type(type_T *actual, type_T *expected, int offset, cctx_T *cctx) - { - if (check_type(expected, actual, FALSE) == OK) - return OK; - if (actual->tt_type != VAR_ANY - && actual->tt_type != VAR_UNKNOWN - && !(actual->tt_type == VAR_FUNC - && (actual->tt_member == &t_any || actual->tt_argcount < 0))) - { - type_mismatch(expected, actual); - return FAIL; - } - generate_TYPECHECK(cctx, expected, offset); - return OK; - } - /* * parse a list: [expr, expr] * "*arg" points to the '['. --- 2708,2713 ---- *************** *** 2734,2740 **** // The function will have one line: "return {expr}". // Compile it into instructions. ! compile_def_function(ufunc, TRUE); if (ufunc->uf_dfunc_idx >= 0) { --- 2776,2782 ---- // The function will have one line: "return {expr}". // Compile it into instructions. ! compile_def_function(ufunc, TRUE, cctx); if (ufunc->uf_dfunc_idx >= 0) { *************** *** 2779,2785 **** // The function will have one line: "return {expr}". // Compile it into instructions. ! compile_def_function(ufunc, TRUE); // compile the arguments *arg = skipwhite(*arg + 1); --- 2821,2827 ---- // The function will have one line: "return {expr}". // Compile it into instructions. ! compile_def_function(ufunc, TRUE, cctx); // compile the arguments *arg = skipwhite(*arg + 1); *************** *** 4227,4240 **** semsg(_("E1017: Variable already declared: %s"), name); goto theend; } ! else { ! if (lvar->lv_const) ! { ! semsg(_("E1018: Cannot assign to a constant: %s"), ! name); ! goto theend; ! } } } else if (STRNCMP(arg, "s:", 2) == 0 --- 4269,4278 ---- semsg(_("E1017: Variable already declared: %s"), name); goto theend; } ! else if (lvar->lv_const) { ! semsg(_("E1018: Cannot assign to a constant: %s"), name); ! goto theend; } } else if (STRNCMP(arg, "s:", 2) == 0 *************** *** 5931,5941 **** * Adds the function to "def_functions". * When "set_return_type" is set then set ufunc->uf_ret_type to the type of the * return statement (used for lambda). * This can be used recursively through compile_lambda(), which may reallocate * "def_functions". */ void ! compile_def_function(ufunc_T *ufunc, int set_return_type) { char_u *line = NULL; char_u *p; --- 5969,5980 ---- * Adds the function to "def_functions". * When "set_return_type" is set then set ufunc->uf_ret_type to the type of the * return statement (used for lambda). + * "outer_cctx" is set for a nested function. * This can be used recursively through compile_lambda(), which may reallocate * "def_functions". */ void ! compile_def_function(ufunc_T *ufunc, int set_return_type, cctx_T *outer_cctx) { char_u *line = NULL; char_u *p; *************** *** 5976,5981 **** --- 6015,6021 ---- CLEAR_FIELD(cctx); cctx.ctx_ufunc = ufunc; cctx.ctx_lnum = -1; + cctx.ctx_outer = outer_cctx; ga_init2(&cctx.ctx_locals, sizeof(lvar_T), 10); ga_init2(&cctx.ctx_type_stack, sizeof(type_T *), 50); ga_init2(&cctx.ctx_imports, sizeof(imported_T), 10); *************** *** 6355,6360 **** --- 6395,6402 ---- dfunc->df_instr = instr->ga_data; dfunc->df_instr_count = instr->ga_len; dfunc->df_varcount = cctx.ctx_locals_count; + if (cctx.ctx_outer_used) + ufunc->uf_flags |= FC_CLOSURE; } { *************** *** 6533,6538 **** --- 6575,6581 ---- case ISN_INDEX: case ISN_JUMP: case ISN_LOAD: + case ISN_LOADOUTER: case ISN_LOADSCRIPT: case ISN_LOADREG: case ISN_LOADV: *** ../vim-8.2.0676/src/proto/vim9compile.pro 2020-04-19 16:28:55.292496003 +0200 --- src/proto/vim9compile.pro 2020-05-01 17:53:14.174715901 +0200 *************** *** 9,15 **** char_u *to_name_const_end(char_u *arg); int assignment_len(char_u *p, int *heredoc); int check_vim9_unlet(char_u *name); ! void compile_def_function(ufunc_T *ufunc, int set_return_type); void delete_instr(isn_T *isn); void delete_def_function(ufunc_T *ufunc); void free_def_functions(void); --- 9,15 ---- char_u *to_name_const_end(char_u *arg); int assignment_len(char_u *p, int *heredoc); int check_vim9_unlet(char_u *name); ! void compile_def_function(ufunc_T *ufunc, int set_return_type, cctx_T *outer_cctx); void delete_instr(isn_T *isn); void delete_def_function(ufunc_T *ufunc); void free_def_functions(void); *** ../vim-8.2.0676/src/userfunc.c 2020-04-27 23:39:26.416849722 +0200 --- src/userfunc.c 2020-05-01 17:55:47.354041020 +0200 *************** *** 14,32 **** #include "vim.h" #if defined(FEAT_EVAL) || defined(PROTO) - // flags used in uf_flags - #define FC_ABORT 0x01 // abort function on error - #define FC_RANGE 0x02 // function accepts range - #define FC_DICT 0x04 // Dict function, uses "self" - #define FC_CLOSURE 0x08 // closure, uses outer scope variables - #define FC_DELETED 0x10 // :delfunction used while uf_refcount > 0 - #define FC_REMOVED 0x20 // function redefined while uf_refcount > 0 - #define FC_SANDBOX 0x40 // function defined in the sandbox - #define FC_DEAD 0x80 // function kept only for reference to dfunc - #define FC_EXPORT 0x100 // "export def Func()" - #define FC_NOARGS 0x200 // no a: variables in lambda - #define FC_VIM9 0x400 // defined in vim9 script file - /* * All user-defined functions are found in this hashtable. */ --- 14,19 ---- *************** *** 3267,3273 **** // ":def Func()" needs to be compiled if (eap->cmdidx == CMD_def) ! compile_def_function(fp, FALSE); goto ret_free; --- 3254,3260 ---- // ":def Func()" needs to be compiled if (eap->cmdidx == CMD_def) ! compile_def_function(fp, FALSE, NULL); goto ret_free; *** ../vim-8.2.0676/src/vim9execute.c 2020-04-30 20:21:36.024020857 +0200 --- src/vim9execute.c 2020-05-01 19:21:29.820169929 +0200 *************** *** 58,63 **** --- 58,66 ---- garray_T ec_stack; // stack of typval_T values int ec_frame; // index in ec_stack: context of ec_dfunc_idx + garray_T *ec_outer_stack; // stack used for closures + int ec_outer_frame; // stack frame in ec_outer_stack + garray_T ec_trystack; // stack of trycmd_T values int ec_in_catch; // when TRUE in catch or finally block *************** *** 229,234 **** --- 232,241 ---- ectx->ec_instr = dfunc->df_instr; estack_push_ufunc(ETYPE_UFUNC, dfunc->df_ufunc, 1); + // used for closures + ectx->ec_outer_stack = ufunc->uf_ectx_stack; + ectx->ec_outer_frame = ufunc->uf_ectx_frame; + // Decide where to start execution, handles optional arguments. init_instr_idx(ufunc, argcount, ectx); *************** *** 508,513 **** --- 515,523 ---- // Get pointer to a local variable on the stack. Negative for arguments. #define STACK_TV_VAR(idx) (((typval_T *)ectx.ec_stack.ga_data) + ectx.ec_frame + STACK_FRAME_SIZE + idx) + // Like STACK_TV_VAR but use the outer scope + #define STACK_OUT_TV_VAR(idx) (((typval_T *)ectx.ec_outer_stack->ga_data) + ectx.ec_outer_frame + STACK_FRAME_SIZE + idx) + CLEAR_FIELD(ectx); ga_init2(&ectx.ec_stack, sizeof(typval_T), 500); if (ga_grow(&ectx.ec_stack, 20) == FAIL) *************** *** 786,791 **** --- 796,810 ---- ++ectx.ec_stack.ga_len; break; + // load variable or argument from outer scope + case ISN_LOADOUTER: + if (ga_grow(&ectx.ec_stack, 1) == FAIL) + goto failed; + copy_tv(STACK_OUT_TV_VAR(iptr->isn_arg.number), + STACK_TV_BOT(0)); + ++ectx.ec_stack.ga_len; + break; + // load v: variable case ISN_LOADV: if (ga_grow(&ectx.ec_stack, 1) == FAIL) *************** *** 1304,1309 **** --- 1323,1336 ---- pt->pt_refcount = 1; ++dfunc->df_ufunc->uf_refcount; + if (dfunc->df_ufunc->uf_flags & FC_CLOSURE) + { + // Closure needs to find local variables in the current + // stack. + dfunc->df_ufunc->uf_ectx_stack = &ectx.ec_stack; + dfunc->df_ufunc->uf_ectx_frame = ectx.ec_frame; + } + if (ga_grow(&ectx.ec_stack, 1) == FAIL) goto failed; tv = STACK_TV_BOT(0); *************** *** 1862,1868 **** checktype_T *ct = &iptr->isn_arg.type; tv = STACK_TV_BOT(ct->ct_off); ! if (tv->v_type != ct->ct_type) { semsg(_("E1029: Expected %s but got %s"), vartype_name(ct->ct_type), --- 1889,1900 ---- checktype_T *ct = &iptr->isn_arg.type; tv = STACK_TV_BOT(ct->ct_off); ! // TODO: better type comparison ! if (tv->v_type != ct->ct_type ! && !((tv->v_type == VAR_PARTIAL ! && ct->ct_type == VAR_FUNC) ! || (tv->v_type == VAR_FUNC ! && ct->ct_type == VAR_PARTIAL))) { semsg(_("E1029: Expected %s but got %s"), vartype_name(ct->ct_type), *************** *** 2029,2040 **** (long long)(iptr->isn_arg.number)); break; case ISN_LOAD: ! if (iptr->isn_arg.number < 0) ! smsg("%4d LOAD arg[%lld]", current, ! (long long)(iptr->isn_arg.number + STACK_FRAME_SIZE)); ! else ! smsg("%4d LOAD $%lld", current, (long long)(iptr->isn_arg.number)); break; case ISN_LOADV: smsg("%4d LOADV v:%s", current, --- 2061,2078 ---- (long long)(iptr->isn_arg.number)); break; case ISN_LOAD: ! case ISN_LOADOUTER: ! { ! char *add = iptr->isn_type == ISN_LOAD ? "" : "OUTER"; ! ! if (iptr->isn_arg.number < 0) ! smsg("%4d LOAD%s arg[%lld]", current, add, ! (long long)(iptr->isn_arg.number ! + STACK_FRAME_SIZE)); ! else ! smsg("%4d LOAD%s $%lld", current, add, (long long)(iptr->isn_arg.number)); + } break; case ISN_LOADV: smsg("%4d LOADV v:%s", current, *** ../vim-8.2.0676/src/structs.h 2020-04-11 20:50:25.376120463 +0200 --- src/structs.h 2020-05-01 17:56:00.257986016 +0200 *************** *** 1561,1567 **** --- 1561,1571 ---- sctx_T uf_script_ctx; // SCTX where function was defined, // used for s: variables int uf_refcount; // reference count, see func_name_refcount() + funccall_T *uf_scoped; // l: local variables for closure + garray_T *uf_ectx_stack; // where compiled closure finds local vars + int uf_ectx_frame; // index of function frame in uf_ectx_stack + char_u *uf_name_exp; // if "uf_name[]" starts with SNR the name with // "" as a string, otherwise NULL char_u uf_name[1]; // name of function (actually longer); can *************** *** 1569,1574 **** --- 1573,1591 ---- // KS_EXTRA KE_SNR) } ufunc_T; + // flags used in uf_flags + #define FC_ABORT 0x01 // abort function on error + #define FC_RANGE 0x02 // function accepts range + #define FC_DICT 0x04 // Dict function, uses "self" + #define FC_CLOSURE 0x08 // closure, uses outer scope variables + #define FC_DELETED 0x10 // :delfunction used while uf_refcount > 0 + #define FC_REMOVED 0x20 // function redefined while uf_refcount > 0 + #define FC_SANDBOX 0x40 // function defined in the sandbox + #define FC_DEAD 0x80 // function kept only for reference to dfunc + #define FC_EXPORT 0x100 // "export def Func()" + #define FC_NOARGS 0x200 // no a: variables in lambda + #define FC_VIM9 0x400 // defined in vim9 script file + #define MAX_FUNC_ARGS 20 // maximum number of function arguments #define VAR_SHORT_LEN 20 // short variable name length #define FIXVAR_CNT 12 // number of fixed variables *** ../vim-8.2.0676/src/vim9.h 2020-04-25 20:02:36.001096124 +0200 --- src/vim9.h 2020-05-01 17:48:14.719774462 +0200 *************** *** 27,32 **** --- 27,33 ---- ISN_LOADW, // push w: variable isn_arg.string ISN_LOADT, // push t: variable isn_arg.string ISN_LOADS, // push s: variable isn_arg.loadstore + ISN_LOADOUTER, // push variable from outer scope isn_arg.number ISN_LOADSCRIPT, // push script-local variable isn_arg.script. ISN_LOADOPT, // push option isn_arg.string ISN_LOADENV, // push environment variable isn_arg.string *** ../vim-8.2.0676/src/testdir/test_vim9_func.vim 2020-04-27 22:47:45.186176148 +0200 --- src/testdir/test_vim9_func.vim 2020-05-01 18:34:46.620912088 +0200 *************** *** 641,644 **** --- 641,653 ---- call assert_equal(1, caught_1059) endfunc + def RefFunc(Ref: func(string): string): string + return Ref('more') + enddef + + def Test_closure_simple() + let local = 'some ' + assert_equal('some more', RefFunc({s -> local .. s})) + enddef + " vim: ts=8 sw=2 sts=2 expandtab tw=80 fdm=marker *** ../vim-8.2.0676/src/version.c 2020-05-01 16:08:08.054859320 +0200 --- src/version.c 2020-05-01 18:35:25.176748724 +0200 *************** *** 748,749 **** --- 748,751 ---- { /* Add new patch number below this line */ + /**/ + 677, /**/ -- Know this story about a nerd who fell into a river and drowned, despite his cries of "F1! F1!"? /// Bram Moolenaar -- Bram@Moolenaar.net -- http://www.Moolenaar.net \\\ /// sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ \\\ \\\ an exciting new programming language -- http://www.Zimbu.org /// \\\ help me help AIDS victims -- http://ICCF-Holland.org ///