How to set up XKB to work with 3 modifier keys and 2 switchable layouts

keyboardkeyboard-layoutx11xkbxorg

Consider the following XKB layout:

xkb_keymap {

xkb_keycodes {
  <ESC> = 9;
  <FK01> = 67;
  <FK02> = 68;
  <FK03> = 69;
  <FK04> = 70;
  <FK05> = 71;
  <FK06> = 72;
  <FK07> = 73;
  <FK08> = 74;
  <FK09> = 75;
  <FK10> = 76;
  <FK11> = 95;
  <FK12> = 96;

  <INS> = 118;
  <DELE> = 119;
  <HOME> = 110;
  <END> = 115;
  <PGUP> = 112;
  <PGDN> = 117;

  <TLDE> = 49;
  <AE01> = 10;
  <AE02> = 11;
  <AE03> = 12;
  <AE04> = 13;
  <AE05> = 14;
  <AE06> = 15;
  <AE07> = 16;
  <AE08> = 17;
  <AE09> = 18;
  <AE10> = 19;
  <AE11> = 20;
  <AE12> = 21;
  <BKSP> = 22;

  <TAB> = 23;
  <AD01> = 24;
  <AD02> = 25;
  <AD03> = 26;
  <AD04> = 27;
  <AD05> = 28;
  <AD06> = 29;
  <AD07> = 30;
  <AD08> = 31;
  <AD09> = 32;
  <AD10> = 33;
  <AD11> = 34;
  <AD12> = 35;

  <BKSL> = 51;

  <CAPS> = 66;
  <AC01> = 38;
  <AC02> = 39;
  <AC03> = 40;
  <AC04> = 41;
  <AC05> = 42;
  <AC06> = 43;
  <AC07> = 44;
  <AC08> = 45;
  <AC09> = 46;
  <AC10> = 47;
  <AC11> = 48;
  <RTRN> = 36;

  <LFSH> = 50;
  <AB01> = 52;
  <AB02> = 53;
  <AB03> = 54;
  <AB04> = 55;
  <AB05> = 56;
  <AB06> = 57;
  <AB07> = 58;
  <AB08> = 59;
  <AB09> = 60;
  <AB10> = 61;
  <RTSH> = 62;

  <LCTL> = 37;
  <LWIN> = 133;
  <LALT> = 64;
  <SPCE> = 65;
  <RALT> = 108;
  <RWIN> = 134;
  <MENU> = 135;
  <RCTL> = 105;

  <UP> = 111;
  <LEFT> = 113;
  <DOWN> = 116;
  <RGHT> = 114;

  <NMLK> = 77;
  <KPDV> = 106;
  <KPMU> = 63;
  <KPSU> = 82;

  <KP7> = 79;
  <KP8> = 80;
  <KP9> = 81;

  <KPAD> = 86;

  <KP4> = 83;
  <KP5> = 84;
  <KP6> = 85;

  <KP1> = 87;
  <KP2> = 88;
  <KP3> = 89;

  <KPEN> = 104;

  <KP0> = 90;
  <KPDL> = 91;
};

xkb_types {
  type "ONE_LEVEL" {
      modifiers = none;
      level_name[Level1] = "Any";
  };
  type "TWO_LEVEL" {
      modifiers = none;
      level_name[Level1] = "Base";
  };
  type "ALPHABETIC" {
      modifiers = none;
      level_name[Level1] = "Base";
  };
  type "KEYPAD" {
      modifiers = none;
      level_name[Level1] = "Base";
  };
  type "MULTI" {
    modifiers = Shift+Mod3+Mod5;
    map[Shift] = Level2;
    map[Mod3] = Level3;
    map[Shift+Mod3] = Level4;
    map[Mod5] = Level5;
    map[Shift+Mod5] = Level6;
    map[Mod3+Mod5] = Level7;
    map[Shift+Mod3+Mod5] = Level8;
    level_name[Level1] = "Base";
    level_name[Level2] = "Shift";
    level_name[Level3] = "Alt";
    level_name[Level4] = "Shift Alt";
    level_name[Level5] = "Control";
    level_name[Level6] = "Control Shift";
    level_name[Level7] = "Control Alt";
    level_name[Level8] = "Control Shift Alt";
  };
};

xkb_compatibility {
  interpret Any + Any{
    action = SetMods(mods=modMapMods);
  };
  interpret ISO_Next_Group {
    action = LockGroup(group=+1);
  };
  indicator "Shift" {
    modifiers = Shift;
  };
};

xkb_symbols {
  key.type = "MULTI";

  key <ESC> {[Escape]};
  key <FK01> {[U2010]}; # hyphen
  key <FK02> {[U2011]}; # non-breaking hyphen
  key <FK03> {[U00AD]}; # soft hyphen
  key <FK04> {[U200B]}; # zero width space
  key <FK05> {[leftdoublequotemark], [guillemotleft]};
  key <FK06> {[rightdoublequotemark], [guillemotright]};
  key <FK07> {[leftsinglequotemark], [doublelowquotemark]};
  key <FK08> {[rightsinglequotemark], [leftdoublequotemark]};
  key <FK09> {[endash]};
  key <FK10> {[emdash]};
  key <FK11> {[ellipsis, U22EF]}; # cdots
  key <FK12> {[nobreakspace, U2060]}; # word joiner
  key <INS> {[Insert]};
  key <DELE> {[Delete]};
  key <HOME> {[Home]};
  key <END> {[End]};
  key <PGUP> {[Prior]};
  key <PGDN> {[Next]};

  key <TLDE> {[grave, asciitilde, dead_grave, dead_tilde, dead_doublegrave, dead_belowtilde]};
  key <AE01> {[1, exclam, U22C1, U22C3, logicalor, union], [exclam, 1]}; # big or, big union
  key <AE02> {[2, at], [at, 2]};
  key <AE03> {[3, numbersign], [slash, 3]};
  key <AE04> {[4, dollar], [semicolon, 4]};
  key <AE05> {[5, percent, dead_horn, dead_hook], [colon, 5]};
  key <AE06> {[6, asciicircum, dead_circumflex, dead_caron, dead_belowcircumflex], [comma, 6]};
  key <AE07> {[7, ampersand, U22C0, U22C2, logicaland, intersection], [period, 7]}; # big and, big intersection
  key <AE08> {[8, asterisk, VoidSymbol, VoidSymbol, multiply, U2297], [question, 8]};
  key <AE09> {[9, parenleft], [parenleft, 9]};
  key <AE10> {[0, parenright, dead_abovering, dead_belowring, emptyset, U2243], [parenright, 0]}; # bar tilde
  key <AE11> {[minus, underscore, dead_macron, dead_belowmacron, U2212, U2245]}; # minus, double bar tilde
  key <AE12> {[equal, plus, dead_breve, dead_belowbreve, notequal, plusminus]};
  key <BKSP> {[BackSpace]};

  key <TAB> {[Tab, ISO_Left_Tab]};
  key <AD01> {[q, Q, Greek_theta, Greek_THETA, U03D1], [Cyrillic_shorti, Cyrillic_SHORTI]}; # vartheta
  key <AD02> {[w, W, a, A, b, B, c, C], [Cyrillic_tse, Cyrillic_TSE]};
  key <AD03> {[e, E, U03F5, Greek_epsilon, elementof, notelementof], [Cyrillic_u, Cyrillic_U]}; # epsilon, varepsilon
  key <AD04> {[r, R, Greek_rho, VoidSymbol, U03F1], [Cyrillic_ka, Cyrillic_KA]}; # varrho
  key <AD05> {[t, T, Greek_tau, VoidSymbol, thorn, THORN], [Cyrillic_ie, Cyrillic_IE]};
  key <AD06> {[y, Y, Greek_upsilon, Greek_UPSILON], [Cyrillic_en, Cyrillic_EN]};
  key <AD07> {[u, U, Greek_psi, Greek_PSI, udiaeresis, Udiaeresis], [Cyrillic_ghe, Cyrillic_GHE]};
  key <AD08> {[i, I, Greek_iota, VoidSymbol, infinity], [Cyrillic_sha, Cyrillic_SHA]};
  key <AD09> {[o, O, Greek_omega, Greek_OMEGA, odiaeresis, Odiaeresis, oe, OE], [Cyrillic_shcha, Cyrillic_SHCHA]};
  key <AD10> {[p, P, Greek_pi, Greek_PI, U03D6], [Cyrillic_ze, Cyrillic_ZE]}; # varpi
  key <AD11> {[bracketleft, braceleft, VoidSymbol, VoidSymbol, leftarrow, U2282], [Cyrillic_ha, Cyrillic_HA]}; # left angle bracket, subset
  key <AD12> {[bracketright, braceright, U21A6, VoidSymbol, rightarrow, U2283], [Cyrillic_hardsign, Cyrillic_HARDSIGN]}; # right angle bracket, supset
  key <BKSL> {[backslash, bar, VoidSymbol, VoidSymbol, U2016]};

  key <CAPS> {[ISO_Next_Group]};
  key <AC01> {[a, A, Greek_alpha, VoidSymbol, adiaeresis, Adiaeresis, ae, AE], [Cyrillic_ef, Cyrillic_EF]};
  key <AC02> {[s, S, Greek_sigma, Greek_SIGMA, ssharp, U03C2], [Cyrillic_yeru, Cyrillic_YERU]}; # varsigma
  key <AC03> {[d, D, Greek_delta, Greek_DELTA, eth, ETH], [Cyrillic_ve, Cyrillic_VE]};
  key <AC04> {[f, F, Greek_phi, Greek_PHI, U03D5], [Cyrillic_a, Cyrillic_A]}; # varphi
  key <AC05> {[g, G, Greek_gamma, Greek_GAMMA], [Cyrillic_pe, Cyrillic_PE]};
  key <AC06> {[h, H, Greek_eta], [Cyrillic_er, Cyrillic_ER]};
  key <AC07> {[j, J, Greek_xi, Greek_XI], [Cyrillic_o, Cyrillic_O]};
  key <AC08> {[k, K, Greek_kappa], [Cyrillic_el, Cyrillic_EL]};
  key <AC09> {[l, L, Greek_lambda, Greek_LAMBDA], [Cyrillic_de, Cyrillic_DE]};
  key <AC10> {[semicolon, colon, dead_diaeresis, dead_belowdiaeresis], [Cyrillic_zhe, Cyrillic_ZHE]};
  key <AC11> {[apostrophe, quotedbl, dead_acute, dead_doubleacute, U2202, U2207], [Cyrillic_e, Cyrillic_E]};         
  key <RTRN> {[Return]};

  key <LFSH> {[Shift_L]};
  key <AB01> {[z, Z, Greek_zeta], [Cyrillic_ya, Cyrillic_YA]};
  key <AB02> {[x, X, Greek_chi], [Cyrillic_che, Cyrillic_CHE]};
  key <AB03> {[c, C, dead_cedilla], [Cyrillic_es, Cyrillic_ES]};
  key <AB04> {[v, V, dead_ogonek], [Cyrillic_em, Cyrillic_EM]};
  key <AB05> {[b, B, Greek_beta], [Cyrillic_i, Cyrillic_I]};
  key <AB06> {[n, N, Greek_nu], [Cyrillic_te, Cyrillic_TE]};
  key <AB07> {[m, M, Greek_mu], [Cyrillic_softsign, Cyrillic_SOFTSIGN]};
  key <AB08> {[comma, less, dead_belowcomma, dead_abovecomma, lessthanequal, U27E8], [Cyrillic_be, Cyrillic_BE]};
  key <AB09> {[period, greater, dead_belowdot, dead_abovedot, greaterthanequal, U27E9], [Cyrillic_yu, Cyrillic_YU]};
  key <AB10> {[slash, question, dead_stroke, dead_invertedbreve, dead_abovereversedcomma, periodcentered], [Cyrillic_io, Cyrillic_IO]};             
  key <RTSH> {[Shift_R]};

  key <LCTL> {[Control_L]};
  key <LWIN> {[Super_L]};
  key <LALT> {[Alt_L]};
  key <SPCE> {[space]};
  key <RALT> {[Alt_R]};
  key <RWIN> {[Super_R]};
  key <MENU> {[Menu]};
  key <RCTL> {[Control_R]};

  key <UP> {[Up]};
  key <LEFT> {[Left]};
  key <DOWN> {[Down]};
  key <RGHT> {[Right]};

  key <NMLK> {[Num_Lock]};
  key <KPDV> {[KP_Divide]};
  key <KPMU> {[KP_Multiply]};
  key <KPSU> {[KP_Subtract]};

  key <KP7> {[Delete]};
  key <KP8> {[End]};
  key <KP9> {[Next]};

  key <KPAD> {[KP_Add]};

  key <KP4> {[KP_4]};
  key <KP5> {[KP_5]};
  key <KP6> {[KP_6]};

  key <KP1> {[KP_1]};
  key <KP2> {[KP_2]};
  key <KP3> {[KP_3]};

  key <KPEN> {[KP_Enter]};

  key <KP0> {[KP_0]};
  key <KPDL> {[KP_Delete]};

  modifier_map Shift { Shift_L, Shift_R };
  modifier_map Lock { ISO_Next_Group };
  modifier_map Control { Control_L };
  modifier_map Mod1 { Alt_L };
  modifier_map Mod2 { Num_Lock };
  modifier_map Mod3 { Alt_R };
  modifier_map Mod4 { Super_L, Super_R };
  modifier_map Mod5 { Control_R };
};

};

The intention is to have two layouts that are switched using Caps Lack; for each layout there are 3 modifier keys: Shift, right Alt, and right Control.
Thus each key is supposed to generate up to 8 different letters in each layout.
For example, the following line

  key <AD02> {[w, W, a, A, b, B, c, C], [Cyrillic_tse, Cyrillic_TSE]};

aims to make the following assignments in the first layout:

w → w
Shift+w → W
Alt_R+w → a
Alt_R+Shift+w → A
Control_R+w → b
Control_R+Shift+w → B
Control_R+Alt_R+w → c
Control_R+Alt_R+Shift+w → C

When I load this layout using xkbcomp – :0,
only 4 out of 8 combinations work as expected,
as follows:

w → w
Shift+w → W
Alt_R+w → a
Alt_R+Shift+w → a
Control_R+w → b
Control_R+Shift+w → b
Control_R+Alt_R+w → b
Control_R+Alt_R+Shift+w → b

In other words, combinations of two or three modifier keys don't work.
What should be changed in order to obtain the desired behavior?

Best Answer

Let's try simplifying this a little.

I'm not aware of any symbols command keys.type=MULTI, so I removed that line first.

I saved your file as keys (minus that line) and ran xkbcomp -w 10 keys. It reports:

Warning:          No automatic type for 6 symbols
                  Using  for the <TLDE> key (keycode 49)
Warning:          Type "" is not defined
                  Using TWO_LEVEL for the <TLDE> key (keycode 49)

(and a lot more errors after that)

So, let's add a definition MULTI3 that mimics MULTI but with 3 (6) levels:

  type "MULTI3" {
    modifiers = Shift+Mod3+Mod5;
    map[Shift] = Level2;
    map[Mod3] = Level3;
    map[Shift+Mod3] = Level4;
    map[Mod5] = Level5;
    map[Shift+Mod5] = Level6;
    level_name[Level1] = "Base";
    level_name[Level2] = "Shift"; 
    level_name[Level3] = "Alt";  
    level_name[Level4] = "Shift Alt";
    level_name[Level5] = "Control";
    level_name[Level6] = "Control Shift";
  };

… and then assign it to TLDE:

  key <TLDE> {type="MULTI3", symbols=[grave, asciitilde, dead_grave, dead_tilde, dead_doublegrave, dead_belowtilde]};

OK, re-running xkbcomp -w 10 keys confirms that this eliminates that error.

The same type of error affects 27 other keys … so, annotating them with the desired type, and/or creating new types to handle them, should eliminate those errors.

The case of having two groups defined (which, by the way, will interact in very unpleasant ways with most tools for picking a keyboard layout, eg in Gnome) in the same file, you will have to specify the symbols and types for each group. I've never tried this myself, but eg:

  key <AD01> {type[group1]="NewTypeFor5Levels",
              symbols[group1]=[q, Q, Greek_theta, Greek_THETA, U03D1], 
              type[group2]="TWO_LEVEL",
              symbols[group2]=[Cyrillic_shorti, Cyrillic_SHORTI]};
       // vartheta
  key <AD02> {type[group1]="MULTI",
              symbols[group1]=[w, W, a, A, b, B, c, C],
              type[group2]="TWO_LEVEL",
              symbols[group2]=[Cyrillic_tse, Cyrillic_TSE]};

Once the file can be compiled, you should have better success.

Related Question