[Gambas-bugtracker] Bug #3116: extend Tokenize with maps

[<bugtracker@xxxxxxxxxxxxxx>]

http://gambaswiki.org/bugtracker/edit?object=BUG.3116&from=L21haW4-

Brian G reported a new bug.

Summary
-------

extend Tokenize with maps

Type             : Request
Priority         : Medium
Gambas version   : Master
Product          : Language


Description
-----------

add a maps parameter to contain the type and position arrays after the parse, also an extra boolean to strip the quotes from quoted text.
and if possible allow the included characters to be at the beginning of a symbol if they do not conflict with an operator
so if the maps option is provided, generate the two arrays.

This would greatly enhance this function, making it useful in many many ways.

public sub Tokenize("source string","additional char to allow in symbols",["array of string identifiers"],[array of operators], include spaces, optional keepStringIdentoifiersas as boolean = true, MapsToReturn as variant[]) as string[]

Public Sub Main()

    Print "Hello world"
    Dim maps As New Variant[]  
    ' when tokenize returns maps contains two arrays, type perhaps matching (highlight.symbol, highlight.operator,highlight.string, highlight.unknown or error) or just 1 2 3 4(maybe have a named enumeration lol)
    ' the second array containing the position of each item found
    ' and perhaps one more boolean to strip the quotes, since we have maps, I would assume you actually have this internally anyway.
    
    Dim x As String[] = Tokenize("$this sil-lty listy_list _buffy 'stuff' + 56 - 100", "$_-", ["\'"], ["+", "=", "-", "_"], False,true, maps)
    Print x.Join(", ")

End


System information
------------------

[System]
Gambas=3.21.99 a4c8eae01 (master)
OperatingSystem=Linux
Distribution=Linux Mint 22.2 (Zara)
Kernel=6.14.0-37-generic
Architecture=x86_64
Cores=8
Memory=13929M
Language=en_US.UTF-8
Platform=x11
Desktop=MATE
DesktopResolution=96
DesktopScale=7
WidgetTheme=vimix-dark-amethyst
Font=Ubuntu Mono,10
DarkTheme=True
[Programs]
appimage-builder=appimage-builder: 1.1.0
dpkg=Debian 'dpkg' package management program version 1.22.6 (amd64).
gcc=gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0
git=git version 2.43.0
msgmerge=msgmerge (GNU gettext-tools) 0.21
rpmbuild=RPM version 4.18.2
[Libraries]
Cairo=libcairo.so.2.11800.0
Curl=libcurl.so.4.8.0
DBus=libdbus-1.so.3.32.4
GDK2=libgdk-x11-2.0.so.0.2400.33
GDK3=libgdk-3.so.0.2409.32
GStreamer=libgstreamer-1.0.so.0.2402.0
GTK+2=libgtk-x11-2.0.so.0.2400.33
GTK+3=libgtk-3.so.0.2409.32
OpenGL=libGL.so.1.7.0
Poppler=libpoppler.so.134.0.0
QT5=libQt5Core.so.5.15.13
QT6=libQt6Core.so.6.4.2
RSvg=librsvg-2.so.2.50.0
SDL=libSDL2-2.0.so.0.3000.0
SQLite3=libsqlite3.so.0.8.6
[Environment]
CLUTTER_BACKEND=x11
COMPIZ_CONFIG_PROFILE=mint
DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1000/bus
DESKTOP_SESSION=mate
DISPLAY=:0
GB_GUI=gb.gtk3
GDMSESSION=mate
GDM_LANG=en_US
GIO_LAUNCHED_DESKTOP_FILE_PID=788790
GPG_AGENT_INFO=/run/user/1000/gnupg/S.gpg-agent:0:1
GTK3_MODULES=xapp-gtk3-module
GTK_MODULES=gail:atk-bridge
GTK_OVERLAY_SCROLLING=0
HOME=<home>
LANG=en_US.UTF-8
LANGUAGE=en_US.UTF-8
LC_ADDRESS=en_US.UTF-8
LC_IDENTIFICATION=en_US.UTF-8
LC_MEASUREMENT=en_US.UTF-8
LC_MONETARY=en_US.UTF-8
LC_NAME=en_US.UTF-8
LC_NUMERIC=en_US.UTF-8
LC_PAPER=en_US.UTF-8
LC_TELEPHONE=en_US.UTF-8
LC_TIME=en_US.UTF-8
LIBVIRT_DEFAULT_URI=qemu:///system
LOGNAME=<user>
MATE_DESKTOP_SESSION_ID=this-is-deprecated
NVM_BIN=<home>/.nvm/versions/node/v24.11.0/bin
NVM_CD_FLAGS=
NVM_DIR=<home>/.nvm
NVM_INC=<home>/.nvm/versions/node/v24.11.0/include/node
PAPERSIZE=a4
PATH=<home>/.nvm/versions/node/v24.11.0/bin:<home>/.config/emacs/bin:<home>/.local/bin:<home>/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/usr/local/go/bin
PICO_SDK_PATH=<home>/Projects/picomite/pico/pico-sdk
PWD=<home>
QT_ACCESSIBILITY=1
QT_FONT_DPI=96
QT_LOGGING_RULES=*.debug=false
QT_SCALE_FACTOR=1
SESSION_MANAGER=local/<hostname>:@/tmp/.ICE-unix/513169,unix/<hostname>:/tmp/.ICE-unix/513169
SHELL=/usr/bin/gsh
SHLVL=0
SSH_AUTH_SOCK=/run/user/1000/keyring/ssh
TZ=:/etc/localtime
USER=<user>
XAUTHORITY=<home>/.Xauthority
XDG_ACTIVATION_TOKEN=mate-panel-513660-<hostname>-gambas3-4_TIME145688231
XDG_CONFIG_DIRS=/etc/xdg/xdg-mate:/etc/xdg
XDG_CURRENT_DESKTOP=MATE
XDG_DATA_DIRS=/usr/share/mate:<home>/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share
XDG_GREETER_DATA_DIR=/var/lib/lightdm-data/<user>
XDG_RUNTIME_DIR=/run/user/1000
XDG_SEAT=seat0
XDG_SEAT_PATH=/org/freedesktop/DisplayManager/Seat0
XDG_SESSION_CLASS=user
XDG_SESSION_DESKTOP=mate
XDG_SESSION_ID=c4
XDG_SESSION_PATH=/org/freedesktop/DisplayManager/Session1
XDG_SESSION_TYPE=x11
XDG_VTNR=7
gshlibrary=<home>/Projects/gshlibrary
gshproject=<home>/Projects/gsh



----[ Gambas bugtracker-list is hosted by https://www.hostsharing.net ]----